@kazedevid/tiktok-sup 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +142 -2
- package/bin/cli.js +108 -0
- package/index.d.ts +189 -0
- package/index.js +630 -0
- package/package.json +41 -3
- package/test.js +74 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 KazeDevID
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,143 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @kazedevid/tiktok-sup 🚀
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **Pure TikTok Web Scraper for Node.js (v1.1.0)** — Scrapes video metadata, direct no-watermark MP4 streams, MP3 audio, user profiles, comments, hashtags, and trending topics directly from `tiktok.com`. **100% pure web scraping with zero 3rd-party APIs, zero external proxies, and zero heavy browser automation.**
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/@kazedevid/tiktok-sup)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
[](https://github.com/KazeDevID/tiktok-sup)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## ✨ Features
|
|
12
|
+
|
|
13
|
+
- 🎯 **100% Pure tiktok.com Scraping** — Zero reliance on rapidapi, tikwm, ssstik, or external proxies.
|
|
14
|
+
- ⚡ **Multi-Strategy Parser** — Extracts mobile reflow data, JSON payloads, and session cookies automatically.
|
|
15
|
+
- 🎬 **No-Watermark & Watermark Streams** — Provides `noWatermarkUrl` (direct playAddr) and standard `watermarkUrl`.
|
|
16
|
+
- 🖼️ **Photo Slideshows** — Full resolution photo carousel images extractor.
|
|
17
|
+
- 🎵 **Soundtrack Audio Extractor** — Direct MP3 play URLs, durations, and album artwork.
|
|
18
|
+
- 👤 **Full User Profile Scraper** — Followers, following, total hearts/likes, video count, bio link, secUid, and verified checkmarks.
|
|
19
|
+
- 💬 **Comments Extractor** — Parses initial comments list directly from the post payload.
|
|
20
|
+
- 📈 **Trending & Discover** — Real-time creators, viral hashtags with view counts, and popular audio tracks.
|
|
21
|
+
- 📥 **Built-in Downloader** — Helpers `downloadFile()` and `downloadBuffer()` for Telegram bots, Discord bots, and Express backends.
|
|
22
|
+
- 📦 **TypeScript Ready** — Includes complete type declarations out of the box (`index.d.ts`).
|
|
23
|
+
- 💻 **CLI Tool Included** — Instant terminal execution via `npx @kazedevid/tiktok-sup`.
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## 📦 Installation
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
npm install @kazedevid/tiktok-sup
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
or with yarn / pnpm / bun:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
yarn add @kazedevid/tiktok-sup
|
|
37
|
+
pnpm add @kazedevid/tiktok-sup
|
|
38
|
+
bun add @kazedevid/tiktok-sup
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## 🚀 Usage Guide
|
|
44
|
+
|
|
45
|
+
### 1. Scrape Video & Download (No Watermark)
|
|
46
|
+
|
|
47
|
+
```javascript
|
|
48
|
+
import { scrapeVideo, downloadFile } from '@kazedevid/tiktok-sup';
|
|
49
|
+
|
|
50
|
+
// Works with full URLs, short links (vt.tiktok.com), or raw video IDs
|
|
51
|
+
const video = await scrapeVideo('https://www.tiktok.com/@zachking/video/6768504823336815877');
|
|
52
|
+
|
|
53
|
+
console.log('Title:', video.description);
|
|
54
|
+
console.log('Author:', video.author.nickname, `(@${video.author.username})`);
|
|
55
|
+
console.log('Views:', video.stats.views.toLocaleString());
|
|
56
|
+
console.log('Likes:', video.stats.likes.toLocaleString());
|
|
57
|
+
console.log('No-Watermark URL:', video.video?.noWatermarkUrl);
|
|
58
|
+
console.log('Audio Track:', video.music?.title, video.music?.playUrl);
|
|
59
|
+
|
|
60
|
+
// Download directly to disk:
|
|
61
|
+
const streamUrl = video.video.noWatermarkUrl || video.video.playUrl;
|
|
62
|
+
await downloadFile(streamUrl, 'video_clean.mp4', video.sessionCookies);
|
|
63
|
+
console.log('Downloaded clean MP4!');
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### 2. Scrape User Profile
|
|
67
|
+
|
|
68
|
+
```javascript
|
|
69
|
+
import { scrapeUser } from '@kazedevid/tiktok-sup';
|
|
70
|
+
|
|
71
|
+
// Pass username with or without '@', or full profile URL
|
|
72
|
+
const user = await scrapeUser('mrbeast');
|
|
73
|
+
|
|
74
|
+
console.log('Name:', user.nickname);
|
|
75
|
+
console.log('Followers:', user.stats.followerCount.toLocaleString());
|
|
76
|
+
console.log('Total Likes:', user.stats.heartCount.toLocaleString());
|
|
77
|
+
console.log('Bio:', user.signature);
|
|
78
|
+
console.log('secUid:', user.secUid);
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### 3. Get Media Buffer (for Bots / Telegram / Discord)
|
|
82
|
+
|
|
83
|
+
```javascript
|
|
84
|
+
import { scrapeVideo, downloadBuffer } from '@kazedevid/tiktok-sup';
|
|
85
|
+
|
|
86
|
+
const video = await scrapeVideo('https://vt.tiktok.com/...');
|
|
87
|
+
const buffer = await downloadBuffer(video.video.playUrl, video.sessionCookies);
|
|
88
|
+
|
|
89
|
+
// Send buffer to Telegram, Discord, or S3
|
|
90
|
+
// bot.sendVideo(chatId, buffer);
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### 4. Trending & Hashtags
|
|
94
|
+
|
|
95
|
+
```javascript
|
|
96
|
+
import { scrapeTrending, scrapeHashtag } from '@kazedevid/tiktok-sup';
|
|
97
|
+
|
|
98
|
+
// Get trending items
|
|
99
|
+
const trending = await scrapeTrending('all'); // 'all' | 'creator' | 'hashtag' | 'music'
|
|
100
|
+
|
|
101
|
+
// Scrape hashtag stats
|
|
102
|
+
const tag = await scrapeHashtag('fyp');
|
|
103
|
+
console.log(tag.title, tag.views);
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 💻 CLI Usage
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# Scrape video info
|
|
112
|
+
npx @kazedevid/tiktok-sup video "https://www.tiktok.com/@zachking/video/6768504823336815877"
|
|
113
|
+
|
|
114
|
+
# Scrape user profile
|
|
115
|
+
npx @kazedevid/tiktok-sup user khaby.lame
|
|
116
|
+
|
|
117
|
+
# List trending creators & sounds
|
|
118
|
+
npx @kazedevid/tiktok-sup trending
|
|
119
|
+
|
|
120
|
+
# Download video directly
|
|
121
|
+
npx @kazedevid/tiktok-sup download "https://vt.tiktok.com/..." clean.mp4
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## 📚 API Reference
|
|
127
|
+
|
|
128
|
+
| Function | Parameters | Returns | Description |
|
|
129
|
+
| :--- | :--- | :--- | :--- |
|
|
130
|
+
| `scrapeVideo(url)` | `url: string` | `Promise<TikTokVideoResult>` | Scrapes full post metadata, no-watermark MP4, audio, and comments |
|
|
131
|
+
| `scrapeUser(username)` | `username: string` | `Promise<TikTokUserProfile>` | Extracts creator metrics, follower count, total likes, bio, and secUid |
|
|
132
|
+
| `scrapeTrending(cat)` | `category?: string` | `Promise<TikTokTrendingItem[]>` | Fetches live trending creators, hashtags, and viral sounds |
|
|
133
|
+
| `scrapeHashtag(tag)` | `tag: string` | `Promise<TikTokHashtagInfo>` | Extracts view counts and info for a hashtag |
|
|
134
|
+
| `scrapeSound(id)` | `soundIdOrUrl: string` | `Promise<TikTokSoundInfo>` | Scrapes soundtrack metadata and play links |
|
|
135
|
+
| `downloadFile(url, out, cookies)` | `url, path, cookies?` | `Promise<string>` | Downloads media buffer directly to a local file |
|
|
136
|
+
| `downloadBuffer(url, cookies)` | `url, cookies?` | `Promise<Buffer>` | Downloads media buffer directly into memory |
|
|
137
|
+
| `fetchMediaStream(url, cookies)` | `url, cookies?` | `Promise<Response>` | Stream response with official TikTok headers and referer |
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## 📄 License
|
|
142
|
+
|
|
143
|
+
MIT © [KazeDevID](https://github.com/KazeDevID)
|
package/bin/cli.js
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* tiktok-sup CLI (v1.1.0)
|
|
5
|
+
* Command line tool for scraping TikTok without 3rd party APIs
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
scrapeVideo,
|
|
10
|
+
scrapeUser,
|
|
11
|
+
scrapeTrending,
|
|
12
|
+
scrapeHashtag,
|
|
13
|
+
scrapeSound,
|
|
14
|
+
downloadFile,
|
|
15
|
+
} from '../index.js';
|
|
16
|
+
|
|
17
|
+
const args = process.argv.slice(2);
|
|
18
|
+
const command = args[0];
|
|
19
|
+
const target = args[1];
|
|
20
|
+
|
|
21
|
+
async function main() {
|
|
22
|
+
if (!command || command === '--help' || command === '-h') {
|
|
23
|
+
console.log(`
|
|
24
|
+
@kazedevid/tiktok-sup - Pure TikTok Web Scraper for Node.js (tiktok.com only)
|
|
25
|
+
|
|
26
|
+
Usage:
|
|
27
|
+
npx @kazedevid/tiktok-sup video <url|id> Scrape video metadata, no-watermark MP4 & audio
|
|
28
|
+
npx @kazedevid/tiktok-sup user <username> Scrape user profile and statistics
|
|
29
|
+
npx @kazedevid/tiktok-sup trending [category] List trending creators, hashtags or music
|
|
30
|
+
npx @kazedevid/tiktok-sup hashtag <tag> Scrape hashtag info and views
|
|
31
|
+
npx @kazedevid/tiktok-sup sound <id|title> Scrape audio track details
|
|
32
|
+
npx @kazedevid/tiktok-sup download <url> [out] Download no-watermark MP4 to file
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
npx @kazedevid/tiktok-sup video 6768504823336815877
|
|
36
|
+
npx @kazedevid/tiktok-sup user mrbeast
|
|
37
|
+
npx @kazedevid/tiktok-sup hashtag fyp
|
|
38
|
+
npx @kazedevid/tiktok-sup download "https://vt.tiktok.com/..." ./myvideo.mp4
|
|
39
|
+
`);
|
|
40
|
+
process.exit(0);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
try {
|
|
44
|
+
if (command === 'video') {
|
|
45
|
+
if (!target) {
|
|
46
|
+
console.error('Error: Please provide a TikTok video URL or ID.');
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
console.log(`Scraping video ${target} directly from tiktok.com...`);
|
|
50
|
+
const data = await scrapeVideo(target);
|
|
51
|
+
console.log(JSON.stringify(data, null, 2));
|
|
52
|
+
} else if (command === 'user') {
|
|
53
|
+
if (!target) {
|
|
54
|
+
console.error('Error: Please provide a TikTok username.');
|
|
55
|
+
process.exit(1);
|
|
56
|
+
}
|
|
57
|
+
console.log(`Scraping profile @${target} directly from tiktok.com...`);
|
|
58
|
+
const data = await scrapeUser(target);
|
|
59
|
+
console.log(JSON.stringify(data, null, 2));
|
|
60
|
+
} else if (command === 'trending') {
|
|
61
|
+
const cat = args[1] || 'all';
|
|
62
|
+
console.log(`Fetching trending items (${cat}) from tiktok.com...`);
|
|
63
|
+
const data = await scrapeTrending(cat);
|
|
64
|
+
console.log(JSON.stringify(data, null, 2));
|
|
65
|
+
} else if (command === 'hashtag') {
|
|
66
|
+
if (!target) {
|
|
67
|
+
console.error('Error: Please provide a hashtag name.');
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
|
70
|
+
const data = await scrapeHashtag(target);
|
|
71
|
+
console.log(JSON.stringify(data, null, 2));
|
|
72
|
+
} else if (command === 'sound') {
|
|
73
|
+
if (!target) {
|
|
74
|
+
console.error('Error: Please provide a sound ID or name.');
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
const data = await scrapeSound(target);
|
|
78
|
+
console.log(JSON.stringify(data, null, 2));
|
|
79
|
+
} else if (command === 'download') {
|
|
80
|
+
if (!target) {
|
|
81
|
+
console.error('Error: Please provide a TikTok video URL or ID.');
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
console.log(`Resolving stream URL for ${target}...`);
|
|
85
|
+
const data = await scrapeVideo(target);
|
|
86
|
+
const streamUrl = data.video?.downloadUrl || data.video?.playUrl;
|
|
87
|
+
if (!streamUrl) {
|
|
88
|
+
console.error('No video stream URL found for this post.');
|
|
89
|
+
process.exit(1);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const outFile = args[2] || `${data.id || 'tiktok_video'}.mp4`;
|
|
93
|
+
console.log(`Downloading stream to ${outFile}...`);
|
|
94
|
+
await downloadFile(streamUrl, outFile, data.sessionCookies);
|
|
95
|
+
console.log(`Saved successfully to ${outFile}!`);
|
|
96
|
+
} else {
|
|
97
|
+
console.error(
|
|
98
|
+
`Unknown command: "${command}". Run "@kazedevid/tiktok-sup --help" for instructions.`
|
|
99
|
+
);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
} catch (err) {
|
|
103
|
+
console.error(`Error: ${err.message}`);
|
|
104
|
+
process.exit(1);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
main();
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
export interface TikTokAuthor {
|
|
2
|
+
id: string;
|
|
3
|
+
username: string;
|
|
4
|
+
nickname: string;
|
|
5
|
+
avatar: string;
|
|
6
|
+
verified: boolean;
|
|
7
|
+
secUid?: string;
|
|
8
|
+
signature?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface TikTokStats {
|
|
12
|
+
views: number;
|
|
13
|
+
likes: number;
|
|
14
|
+
comments: number;
|
|
15
|
+
shares: number;
|
|
16
|
+
bookmarks: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface TikTokBitrate {
|
|
20
|
+
gearName?: string;
|
|
21
|
+
bitrate?: number;
|
|
22
|
+
qualityType?: number;
|
|
23
|
+
codecType?: string;
|
|
24
|
+
url?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface TikTokVideoMedia {
|
|
28
|
+
id: string;
|
|
29
|
+
duration: number;
|
|
30
|
+
width: number;
|
|
31
|
+
height: number;
|
|
32
|
+
ratio?: string;
|
|
33
|
+
cover: string;
|
|
34
|
+
dynamicCover?: string;
|
|
35
|
+
playUrl?: string;
|
|
36
|
+
noWatermarkUrl?: string;
|
|
37
|
+
downloadUrl?: string;
|
|
38
|
+
watermarkUrl?: string;
|
|
39
|
+
bitrates?: TikTokBitrate[];
|
|
40
|
+
format?: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface TikTokMusic {
|
|
44
|
+
id: string;
|
|
45
|
+
title: string;
|
|
46
|
+
author: string;
|
|
47
|
+
duration: number;
|
|
48
|
+
playUrl?: string;
|
|
49
|
+
cover?: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface TikTokComment {
|
|
53
|
+
cid: string;
|
|
54
|
+
text: string;
|
|
55
|
+
createTime: number;
|
|
56
|
+
diggCount: number;
|
|
57
|
+
replyCount: number;
|
|
58
|
+
user: {
|
|
59
|
+
id: string;
|
|
60
|
+
username: string;
|
|
61
|
+
nickname: string;
|
|
62
|
+
avatar: string;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface TikTokVideoResult {
|
|
67
|
+
id: string;
|
|
68
|
+
url: string;
|
|
69
|
+
description: string;
|
|
70
|
+
createTime: number;
|
|
71
|
+
createTimeFormatted: string;
|
|
72
|
+
author: TikTokAuthor;
|
|
73
|
+
stats: TikTokStats;
|
|
74
|
+
video?: TikTokVideoMedia;
|
|
75
|
+
music?: TikTokMusic;
|
|
76
|
+
isPhoto: boolean;
|
|
77
|
+
photos?: string[];
|
|
78
|
+
comments?: TikTokComment[];
|
|
79
|
+
sessionCookies?: string;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface TikTokUserProfile {
|
|
83
|
+
id: string;
|
|
84
|
+
uniqueId: string;
|
|
85
|
+
nickname: string;
|
|
86
|
+
avatar: string;
|
|
87
|
+
avatarMedium: string;
|
|
88
|
+
avatarThumb: string;
|
|
89
|
+
signature: string;
|
|
90
|
+
bioLink?: { link: string; risk: number };
|
|
91
|
+
verified: boolean;
|
|
92
|
+
secUid: string;
|
|
93
|
+
privateAccount: boolean;
|
|
94
|
+
stats: {
|
|
95
|
+
followerCount: number;
|
|
96
|
+
followingCount: number;
|
|
97
|
+
heartCount: number;
|
|
98
|
+
videoCount: number;
|
|
99
|
+
friendCount: number;
|
|
100
|
+
};
|
|
101
|
+
shareMeta?: {
|
|
102
|
+
title: string;
|
|
103
|
+
desc: string;
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface TikTokTrendingItem {
|
|
108
|
+
type: 'creator' | 'hashtag' | 'music' | 'unknown';
|
|
109
|
+
id: string;
|
|
110
|
+
title: string;
|
|
111
|
+
subTitle?: string;
|
|
112
|
+
desc?: string;
|
|
113
|
+
cover?: string;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export interface TikTokHashtagInfo {
|
|
117
|
+
id: string;
|
|
118
|
+
title: string;
|
|
119
|
+
views?: string;
|
|
120
|
+
desc?: string;
|
|
121
|
+
cover?: string;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export interface TikTokSoundInfo {
|
|
125
|
+
id: string;
|
|
126
|
+
title: string;
|
|
127
|
+
author?: string;
|
|
128
|
+
playUrl?: string;
|
|
129
|
+
cover?: string;
|
|
130
|
+
duration?: number;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export declare function parseTikTokUrl(inputUrl: string): {
|
|
134
|
+
videoId?: string;
|
|
135
|
+
username?: string;
|
|
136
|
+
isShort: boolean;
|
|
137
|
+
isPhoto: boolean;
|
|
138
|
+
normalizedUrl: string;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
export declare function resolveUrl(inputUrl: string): Promise<{
|
|
142
|
+
canonicalUrl: string;
|
|
143
|
+
cookies: string;
|
|
144
|
+
videoId?: string;
|
|
145
|
+
}>;
|
|
146
|
+
|
|
147
|
+
export declare function scrapeVideo(inputUrl: string): Promise<TikTokVideoResult>;
|
|
148
|
+
|
|
149
|
+
export declare function scrapeUser(inputUsername: string): Promise<TikTokUserProfile>;
|
|
150
|
+
|
|
151
|
+
export declare function scrapeTrending(
|
|
152
|
+
category?: 'all' | 'creator' | 'hashtag' | 'music'
|
|
153
|
+
): Promise<TikTokTrendingItem[]>;
|
|
154
|
+
|
|
155
|
+
export declare function scrapeHashtag(tag: string): Promise<TikTokHashtagInfo>;
|
|
156
|
+
|
|
157
|
+
export declare function scrapeSound(soundIdOrUrl: string): Promise<TikTokSoundInfo>;
|
|
158
|
+
|
|
159
|
+
export declare function fetchMediaStream(
|
|
160
|
+
mediaUrl: string,
|
|
161
|
+
cookieStr?: string,
|
|
162
|
+
rangeHeader?: string
|
|
163
|
+
): Promise<Response>;
|
|
164
|
+
|
|
165
|
+
export declare function downloadBuffer(
|
|
166
|
+
mediaUrl: string,
|
|
167
|
+
cookieStr?: string
|
|
168
|
+
): Promise<Buffer>;
|
|
169
|
+
|
|
170
|
+
export declare function downloadFile(
|
|
171
|
+
mediaUrl: string,
|
|
172
|
+
outputPath: string,
|
|
173
|
+
cookieStr?: string
|
|
174
|
+
): Promise<string>;
|
|
175
|
+
|
|
176
|
+
declare const _default: {
|
|
177
|
+
scrapeVideo: typeof scrapeVideo;
|
|
178
|
+
scrapeUser: typeof scrapeUser;
|
|
179
|
+
scrapeTrending: typeof scrapeTrending;
|
|
180
|
+
scrapeHashtag: typeof scrapeHashtag;
|
|
181
|
+
scrapeSound: typeof scrapeSound;
|
|
182
|
+
parseTikTokUrl: typeof parseTikTokUrl;
|
|
183
|
+
resolveUrl: typeof resolveUrl;
|
|
184
|
+
fetchMediaStream: typeof fetchMediaStream;
|
|
185
|
+
downloadBuffer: typeof downloadBuffer;
|
|
186
|
+
downloadFile: typeof downloadFile;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
export default _default;
|
package/index.js
ADDED
|
@@ -0,0 +1,630 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @kazedevid/tiktok-sup (v1.1.0)
|
|
3
|
+
* Pure TikTok web scraper for Node.js without any 3rd party APIs.
|
|
4
|
+
* Scrapes videos (no-watermark), users, audio, comments, hashtags & trending data directly from tiktok.com.
|
|
5
|
+
*
|
|
6
|
+
* @author KazeDevID
|
|
7
|
+
* @license MIT
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import fs from 'fs';
|
|
11
|
+
import path from 'path';
|
|
12
|
+
|
|
13
|
+
export const MOBILE_USER_AGENT =
|
|
14
|
+
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1';
|
|
15
|
+
|
|
16
|
+
export const DESKTOP_USER_AGENT =
|
|
17
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Parses and normalizes any TikTok URL or ID format
|
|
21
|
+
* @param {string} inputUrl
|
|
22
|
+
* @returns {{ videoId?: string, username?: string, isShort: boolean, isPhoto: boolean, normalizedUrl: string }}
|
|
23
|
+
*/
|
|
24
|
+
export function parseTikTokUrl(inputUrl) {
|
|
25
|
+
const clean = String(inputUrl || '').trim();
|
|
26
|
+
|
|
27
|
+
// 1. Raw numeric video ID (e.g. 6768504823336815877)
|
|
28
|
+
if (/^\d{15,22}$/.test(clean)) {
|
|
29
|
+
return {
|
|
30
|
+
videoId: clean,
|
|
31
|
+
isShort: false,
|
|
32
|
+
isPhoto: false,
|
|
33
|
+
normalizedUrl: `https://www.tiktok.com/@tiktok/video/${clean}`,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const isShort =
|
|
38
|
+
clean.includes('vt.tiktok.com') ||
|
|
39
|
+
clean.includes('vm.tiktok.com') ||
|
|
40
|
+
clean.includes('/t/');
|
|
41
|
+
const isPhoto = clean.includes('/photo/');
|
|
42
|
+
|
|
43
|
+
const videoMatch = clean.match(/\/video\/(\d{15,22})/);
|
|
44
|
+
const photoMatch = clean.match(/\/photo\/(\d{15,22})/);
|
|
45
|
+
const vMatch = clean.match(/\/v\/(\d{15,22})/);
|
|
46
|
+
const userMatch = clean.match(/@([a-zA-Z0-9_.-]+)/);
|
|
47
|
+
|
|
48
|
+
const videoId = videoMatch?.[1] || photoMatch?.[1] || vMatch?.[1];
|
|
49
|
+
const username = userMatch?.[1];
|
|
50
|
+
|
|
51
|
+
let normalizedUrl = clean;
|
|
52
|
+
if (!clean.startsWith('http://') && !clean.startsWith('https://')) {
|
|
53
|
+
normalizedUrl = 'https://' + clean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (videoId && !username && !isShort) {
|
|
57
|
+
normalizedUrl = `https://www.tiktok.com/@tiktok/video/${videoId}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
videoId,
|
|
62
|
+
username,
|
|
63
|
+
isShort,
|
|
64
|
+
isPhoto,
|
|
65
|
+
normalizedUrl,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Resolves short links (vt.tiktok.com, vm.tiktok.com) and returns canonical URL and cookies
|
|
71
|
+
* @param {string} inputUrl
|
|
72
|
+
* @returns {Promise<{ canonicalUrl: string, cookies: string, videoId?: string }>}
|
|
73
|
+
*/
|
|
74
|
+
export async function resolveUrl(inputUrl) {
|
|
75
|
+
const { normalizedUrl, videoId: parsedId } = parseTikTokUrl(inputUrl);
|
|
76
|
+
|
|
77
|
+
const res = await fetch(normalizedUrl, {
|
|
78
|
+
headers: {
|
|
79
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
80
|
+
'Accept':
|
|
81
|
+
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
82
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
83
|
+
},
|
|
84
|
+
redirect: 'follow',
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
const cookiesList = (
|
|
88
|
+
res.headers.getSetCookie
|
|
89
|
+
? res.headers.getSetCookie()
|
|
90
|
+
: [res.headers.get('set-cookie')]
|
|
91
|
+
).filter(Boolean);
|
|
92
|
+
const cookies = cookiesList.map((c) => c.split(';')[0]).join('; ');
|
|
93
|
+
|
|
94
|
+
let finalUrl = res.url;
|
|
95
|
+
const idMatch = finalUrl.match(/\/video\/(\d{15,22})/);
|
|
96
|
+
const photoMatch = finalUrl.match(/\/photo\/(\d{15,22})/);
|
|
97
|
+
const extractedId = idMatch?.[1] || photoMatch?.[1] || parsedId;
|
|
98
|
+
|
|
99
|
+
if (finalUrl.includes('tiktok.com/?') && extractedId) {
|
|
100
|
+
finalUrl = `https://www.tiktok.com/@tiktok/video/${extractedId}`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
canonicalUrl: finalUrl,
|
|
105
|
+
cookies,
|
|
106
|
+
videoId: extractedId,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Scrapes a TikTok video or photo post directly from tiktok.com
|
|
112
|
+
* @param {string} inputUrl
|
|
113
|
+
* @returns {Promise<Object>}
|
|
114
|
+
*/
|
|
115
|
+
export async function scrapeVideo(inputUrl) {
|
|
116
|
+
const { canonicalUrl, cookies: initialCookies, videoId: resolvedId } =
|
|
117
|
+
await resolveUrl(inputUrl);
|
|
118
|
+
|
|
119
|
+
const res = await fetch(canonicalUrl, {
|
|
120
|
+
headers: {
|
|
121
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
122
|
+
'Accept':
|
|
123
|
+
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
124
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
125
|
+
'Referer': 'https://www.tiktok.com/',
|
|
126
|
+
...(initialCookies ? { Cookie: initialCookies } : {}),
|
|
127
|
+
},
|
|
128
|
+
redirect: 'follow',
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
if (!res.ok && res.status !== 200) {
|
|
132
|
+
throw new Error(
|
|
133
|
+
`Failed to fetch video page from tiktok.com: HTTP ${res.status}`
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const pageCookiesList = (
|
|
138
|
+
res.headers.getSetCookie
|
|
139
|
+
? res.headers.getSetCookie()
|
|
140
|
+
: [res.headers.get('set-cookie')]
|
|
141
|
+
).filter(Boolean);
|
|
142
|
+
const pageCookies = pageCookiesList.map((c) => c.split(';')[0]).join('; ');
|
|
143
|
+
const combinedCookies = [initialCookies, pageCookies]
|
|
144
|
+
.filter(Boolean)
|
|
145
|
+
.join('; ');
|
|
146
|
+
|
|
147
|
+
const html = await res.text();
|
|
148
|
+
|
|
149
|
+
let itemStruct = null;
|
|
150
|
+
let commentsRaw = [];
|
|
151
|
+
|
|
152
|
+
// Parse Method A: <script id="api-data">
|
|
153
|
+
const apiDataMatch = html.match(
|
|
154
|
+
/<script id="api-data"[^>]*>([\s\S]*?)<\/script>/
|
|
155
|
+
);
|
|
156
|
+
if (apiDataMatch) {
|
|
157
|
+
try {
|
|
158
|
+
const parsed = JSON.parse(apiDataMatch[1]);
|
|
159
|
+
if (parsed.videoDetail?.itemInfo?.itemStruct) {
|
|
160
|
+
itemStruct = parsed.videoDetail.itemInfo.itemStruct;
|
|
161
|
+
}
|
|
162
|
+
if (
|
|
163
|
+
parsed.commentList?.comments &&
|
|
164
|
+
Array.isArray(parsed.commentList.comments)
|
|
165
|
+
) {
|
|
166
|
+
commentsRaw = parsed.commentList.comments;
|
|
167
|
+
}
|
|
168
|
+
} catch (_) {}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Parse Method B: <script id="__UNIVERSAL_DATA_FOR_REHYDRATION__">
|
|
172
|
+
if (!itemStruct) {
|
|
173
|
+
const rehydrationMatch = html.match(
|
|
174
|
+
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
175
|
+
);
|
|
176
|
+
if (rehydrationMatch) {
|
|
177
|
+
try {
|
|
178
|
+
const parsed = JSON.parse(rehydrationMatch[1]);
|
|
179
|
+
const scope = parsed['__DEFAULT_SCOPE__'] || {};
|
|
180
|
+
const videoDetail = scope['webapp.video-detail'];
|
|
181
|
+
if (videoDetail?.itemInfo?.itemStruct) {
|
|
182
|
+
itemStruct = videoDetail.itemInfo.itemStruct;
|
|
183
|
+
}
|
|
184
|
+
} catch (_) {}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Parse Method C: SIGI_STATE
|
|
189
|
+
if (!itemStruct) {
|
|
190
|
+
const sigiMatch = html.match(
|
|
191
|
+
/<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/
|
|
192
|
+
);
|
|
193
|
+
if (sigiMatch) {
|
|
194
|
+
try {
|
|
195
|
+
const parsed = JSON.parse(sigiMatch[1]);
|
|
196
|
+
const items = parsed.ItemModule || {};
|
|
197
|
+
const firstKey = Object.keys(items)[0];
|
|
198
|
+
if (firstKey && items[firstKey]) {
|
|
199
|
+
itemStruct = items[firstKey];
|
|
200
|
+
}
|
|
201
|
+
} catch (_) {}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Parse Method D: Fallback with generic handle
|
|
206
|
+
if (!itemStruct && resolvedId) {
|
|
207
|
+
try {
|
|
208
|
+
const fallbackUrl = `https://www.tiktok.com/@tiktok/video/${resolvedId}`;
|
|
209
|
+
const fbRes = await fetch(fallbackUrl, {
|
|
210
|
+
headers: {
|
|
211
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
212
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
213
|
+
'Referer': 'https://www.tiktok.com/',
|
|
214
|
+
},
|
|
215
|
+
redirect: 'follow',
|
|
216
|
+
});
|
|
217
|
+
if (fbRes.ok) {
|
|
218
|
+
const fbHtml = await fbRes.text();
|
|
219
|
+
const fbApiMatch = fbHtml.match(/<script id="api-data"[^>]*>([\s\S]*?)<\/script>/);
|
|
220
|
+
if (fbApiMatch) {
|
|
221
|
+
const parsed = JSON.parse(fbApiMatch[1]);
|
|
222
|
+
if (parsed.videoDetail?.itemInfo?.itemStruct) {
|
|
223
|
+
itemStruct = parsed.videoDetail.itemInfo.itemStruct;
|
|
224
|
+
}
|
|
225
|
+
if (parsed.commentList?.comments) {
|
|
226
|
+
commentsRaw = parsed.commentList.comments;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
} catch (_) {}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (!itemStruct) {
|
|
234
|
+
throw new Error(
|
|
235
|
+
'Could not locate TikTok video data on tiktok.com. Video may be private, region-restricted, or removed.'
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Photos
|
|
240
|
+
const photos = [];
|
|
241
|
+
if (
|
|
242
|
+
itemStruct.imagePost?.images &&
|
|
243
|
+
Array.isArray(itemStruct.imagePost.images)
|
|
244
|
+
) {
|
|
245
|
+
for (const img of itemStruct.imagePost.images) {
|
|
246
|
+
const u =
|
|
247
|
+
img.displayImage?.urlList?.[0] ||
|
|
248
|
+
img.thumbnail?.urlList?.[0] ||
|
|
249
|
+
img.imageURL?.urlList?.[0];
|
|
250
|
+
if (u) photos.push(u);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const author = {
|
|
255
|
+
id: String(itemStruct.author?.id || ''),
|
|
256
|
+
username: String(itemStruct.author?.uniqueId || ''),
|
|
257
|
+
nickname: String(itemStruct.author?.nickname || ''),
|
|
258
|
+
avatar: String(
|
|
259
|
+
itemStruct.author?.avatarLarger ||
|
|
260
|
+
itemStruct.author?.avatarMedium ||
|
|
261
|
+
itemStruct.author?.avatarThumb ||
|
|
262
|
+
''
|
|
263
|
+
),
|
|
264
|
+
verified: Boolean(itemStruct.author?.verified),
|
|
265
|
+
secUid: itemStruct.author?.secUid,
|
|
266
|
+
signature: itemStruct.author?.signature,
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
const stats = {
|
|
270
|
+
views: Number(itemStruct.stats?.playCount || 0),
|
|
271
|
+
likes: Number(itemStruct.stats?.diggCount || 0),
|
|
272
|
+
comments: Number(itemStruct.stats?.commentCount || 0),
|
|
273
|
+
shares: Number(itemStruct.stats?.shareCount || 0),
|
|
274
|
+
bookmarks: Number(itemStruct.stats?.collectCount || 0),
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
const bitrates = [];
|
|
278
|
+
if (Array.isArray(itemStruct.video?.bitrateInfo)) {
|
|
279
|
+
for (const b of itemStruct.video.bitrateInfo) {
|
|
280
|
+
bitrates.push({
|
|
281
|
+
gearName: b.GearName,
|
|
282
|
+
bitrate: b.Bitrate,
|
|
283
|
+
qualityType: b.QualityType,
|
|
284
|
+
codecType: b.CodecType,
|
|
285
|
+
url: b.PlayAddr?.UrlList?.[0],
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
let video = undefined;
|
|
291
|
+
if (itemStruct.video) {
|
|
292
|
+
const playAddr = itemStruct.video.playAddr
|
|
293
|
+
? String(itemStruct.video.playAddr)
|
|
294
|
+
: undefined;
|
|
295
|
+
const downloadAddr = itemStruct.video.downloadAddr
|
|
296
|
+
? String(itemStruct.video.downloadAddr)
|
|
297
|
+
: playAddr;
|
|
298
|
+
|
|
299
|
+
video = {
|
|
300
|
+
id: String(itemStruct.video.id || itemStruct.id || ''),
|
|
301
|
+
duration: Number(itemStruct.video.duration || 0),
|
|
302
|
+
width: Number(itemStruct.video.width || 0),
|
|
303
|
+
height: Number(itemStruct.video.height || 0),
|
|
304
|
+
ratio: itemStruct.video.ratio,
|
|
305
|
+
cover: String(itemStruct.video.cover || ''),
|
|
306
|
+
dynamicCover: itemStruct.video.dynamicCover
|
|
307
|
+
? String(itemStruct.video.dynamicCover)
|
|
308
|
+
: undefined,
|
|
309
|
+
playUrl: playAddr,
|
|
310
|
+
noWatermarkUrl: playAddr, // Direct high-quality clean stream
|
|
311
|
+
downloadUrl: downloadAddr,
|
|
312
|
+
watermarkUrl: downloadAddr,
|
|
313
|
+
bitrates: bitrates.length > 0 ? bitrates : undefined,
|
|
314
|
+
format: itemStruct.video.format,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
let music = undefined;
|
|
319
|
+
if (itemStruct.music) {
|
|
320
|
+
music = {
|
|
321
|
+
id: String(itemStruct.music.id || ''),
|
|
322
|
+
title: String(itemStruct.music.title || 'Original Sound'),
|
|
323
|
+
author: String(itemStruct.music.authorName || author.nickname),
|
|
324
|
+
duration: Number(itemStruct.music.duration || 0),
|
|
325
|
+
playUrl: itemStruct.music.playUrl
|
|
326
|
+
? String(itemStruct.music.playUrl)
|
|
327
|
+
: undefined,
|
|
328
|
+
cover: itemStruct.music.coverLarge
|
|
329
|
+
? String(itemStruct.music.coverLarge)
|
|
330
|
+
: itemStruct.music.coverMedium
|
|
331
|
+
? String(itemStruct.music.coverMedium)
|
|
332
|
+
: undefined,
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const comments = commentsRaw.map((c) => ({
|
|
337
|
+
cid: String(c.cid || ''),
|
|
338
|
+
text: String(c.text || ''),
|
|
339
|
+
createTime: Number(c.create_time || 0),
|
|
340
|
+
diggCount: Number(c.digg_count || 0),
|
|
341
|
+
replyCount: Number(c.reply_comment_total || 0),
|
|
342
|
+
user: {
|
|
343
|
+
id: String(c.user?.uid || ''),
|
|
344
|
+
username: String(c.user?.unique_id || ''),
|
|
345
|
+
nickname: String(c.user?.nickname || ''),
|
|
346
|
+
avatar: String(c.user?.avatar_thumb?.url_list?.[0] || ''),
|
|
347
|
+
},
|
|
348
|
+
}));
|
|
349
|
+
|
|
350
|
+
const createTime = Number(itemStruct.createTime || 0);
|
|
351
|
+
const createDate = createTime > 0 ? new Date(createTime * 1000) : new Date();
|
|
352
|
+
|
|
353
|
+
return {
|
|
354
|
+
id: String(itemStruct.id || resolvedId || ''),
|
|
355
|
+
url: canonicalUrl,
|
|
356
|
+
description: String(itemStruct.desc || ''),
|
|
357
|
+
createTime,
|
|
358
|
+
createTimeFormatted: createDate.toLocaleString('en-US', {
|
|
359
|
+
year: 'numeric',
|
|
360
|
+
month: 'short',
|
|
361
|
+
day: 'numeric',
|
|
362
|
+
hour: '2-digit',
|
|
363
|
+
minute: '2-digit',
|
|
364
|
+
}),
|
|
365
|
+
author,
|
|
366
|
+
stats,
|
|
367
|
+
video,
|
|
368
|
+
music,
|
|
369
|
+
isPhoto: photos.length > 0,
|
|
370
|
+
photos: photos.length > 0 ? photos : undefined,
|
|
371
|
+
comments: comments.length > 0 ? comments : undefined,
|
|
372
|
+
sessionCookies: combinedCookies,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Scrapes a TikTok user profile directly from tiktok.com
|
|
378
|
+
* @param {string} inputUsername
|
|
379
|
+
* @returns {Promise<Object>}
|
|
380
|
+
*/
|
|
381
|
+
export async function scrapeUser(inputUsername) {
|
|
382
|
+
const clean = String(inputUsername || '')
|
|
383
|
+
.trim()
|
|
384
|
+
.replace(/^https?:\/\/(www\.)?tiktok\.com\/@?/, '')
|
|
385
|
+
.replace(/^@+/, '')
|
|
386
|
+
.split('/')[0]
|
|
387
|
+
.split('?')[0];
|
|
388
|
+
|
|
389
|
+
if (!clean) {
|
|
390
|
+
throw new Error(
|
|
391
|
+
'Please provide a valid TikTok username (e.g. "khaby.lame" or "@mrbeast").'
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const url = `https://www.tiktok.com/@${encodeURIComponent(clean)}`;
|
|
396
|
+
const res = await fetch(url, {
|
|
397
|
+
headers: {
|
|
398
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
399
|
+
'Accept':
|
|
400
|
+
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
401
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
402
|
+
},
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
if (!res.ok) {
|
|
406
|
+
throw new Error(`Failed to fetch user @${clean}: HTTP ${res.status}`);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
const html = await res.text();
|
|
410
|
+
const match = html.match(
|
|
411
|
+
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
412
|
+
);
|
|
413
|
+
|
|
414
|
+
if (!match) {
|
|
415
|
+
throw new Error(
|
|
416
|
+
`Could not parse user @${clean} data from tiktok.com.`
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const data = JSON.parse(match[1]);
|
|
421
|
+
const userDetail = data['__DEFAULT_SCOPE__']?.['webapp.user-detail'];
|
|
422
|
+
if (!userDetail?.userInfo?.user) {
|
|
423
|
+
throw new Error(
|
|
424
|
+
`TikTok user @${clean} does not exist or is unavailable.`
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
const u = userDetail.userInfo.user;
|
|
429
|
+
const s = userDetail.userInfo.stats || {};
|
|
430
|
+
|
|
431
|
+
return {
|
|
432
|
+
id: String(u.id || ''),
|
|
433
|
+
uniqueId: String(u.uniqueId || clean),
|
|
434
|
+
nickname: String(u.nickname || clean),
|
|
435
|
+
avatar: String(u.avatarLarger || u.avatarMedium || u.avatarThumb || ''),
|
|
436
|
+
avatarMedium: String(u.avatarMedium || u.avatarThumb || ''),
|
|
437
|
+
avatarThumb: String(u.avatarThumb || ''),
|
|
438
|
+
signature: String(u.signature || ''),
|
|
439
|
+
bioLink: u.bioLink,
|
|
440
|
+
verified: Boolean(u.verified),
|
|
441
|
+
secUid: String(u.secUid || ''),
|
|
442
|
+
privateAccount: Boolean(u.privateAccount),
|
|
443
|
+
stats: {
|
|
444
|
+
followerCount: Number(s.followerCount || 0),
|
|
445
|
+
followingCount: Number(s.followingCount || 0),
|
|
446
|
+
heartCount: Number(s.heartCount || s.heart || 0),
|
|
447
|
+
videoCount: Number(s.videoCount || 0),
|
|
448
|
+
friendCount: Number(s.friendCount || 0),
|
|
449
|
+
},
|
|
450
|
+
shareMeta: userDetail.shareMeta,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Scrapes trending creators, hashtags, and sound tracks directly from tiktok.com
|
|
456
|
+
* @param {'all' | 'creator' | 'hashtag' | 'music'} [category]
|
|
457
|
+
* @returns {Promise<Array<Object>>}
|
|
458
|
+
*/
|
|
459
|
+
export async function scrapeTrending(category = 'all') {
|
|
460
|
+
const res = await fetch('https://www.tiktok.com/node/share/discover', {
|
|
461
|
+
headers: {
|
|
462
|
+
'User-Agent': DESKTOP_USER_AGENT,
|
|
463
|
+
'Accept': 'application/json, text/plain, */*',
|
|
464
|
+
'Referer': 'https://www.tiktok.com/',
|
|
465
|
+
},
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
if (!res.ok) {
|
|
469
|
+
throw new Error(`Failed to fetch trending discovery data: HTTP ${res.status}`);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const json = await res.json();
|
|
473
|
+
const results = [];
|
|
474
|
+
|
|
475
|
+
if (Array.isArray(json.body)) {
|
|
476
|
+
for (const group of json.body) {
|
|
477
|
+
if (Array.isArray(group.exploreList)) {
|
|
478
|
+
for (const item of group.exploreList) {
|
|
479
|
+
const card = item.cardItem;
|
|
480
|
+
if (!card) continue;
|
|
481
|
+
|
|
482
|
+
let type = 'unknown';
|
|
483
|
+
if (card.type === 2) type = 'creator';
|
|
484
|
+
else if (card.type === 3) type = 'hashtag';
|
|
485
|
+
else if (card.type === 1) type = 'music';
|
|
486
|
+
|
|
487
|
+
if (category && category !== 'all' && type !== category) {
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
results.push({
|
|
492
|
+
type,
|
|
493
|
+
id: String(card.id || ''),
|
|
494
|
+
title: String(card.title || ''),
|
|
495
|
+
subTitle: card.subTitle ? String(card.subTitle) : undefined,
|
|
496
|
+
desc: card.desc ? String(card.desc) : undefined,
|
|
497
|
+
cover: card.cover ? String(card.cover) : undefined,
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
return results;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Scrapes hashtag info
|
|
509
|
+
* @param {string} tag
|
|
510
|
+
* @returns {Promise<Object>}
|
|
511
|
+
*/
|
|
512
|
+
export async function scrapeHashtag(tag) {
|
|
513
|
+
const cleanTag = String(tag || '').trim().replace(/^#+/, '');
|
|
514
|
+
const trending = await scrapeTrending('hashtag');
|
|
515
|
+
const matched = trending.find(
|
|
516
|
+
(t) => t.title.toLowerCase().replace(/^#+/, '') === cleanTag.toLowerCase()
|
|
517
|
+
);
|
|
518
|
+
|
|
519
|
+
if (matched) {
|
|
520
|
+
return {
|
|
521
|
+
id: matched.id,
|
|
522
|
+
title: matched.title,
|
|
523
|
+
views: matched.subTitle,
|
|
524
|
+
desc: matched.desc,
|
|
525
|
+
cover: matched.cover,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
return {
|
|
530
|
+
id: cleanTag,
|
|
531
|
+
title: `#${cleanTag}`,
|
|
532
|
+
views: 'Trending',
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Scrapes sound track details
|
|
538
|
+
* @param {string} soundIdOrUrl
|
|
539
|
+
* @returns {Promise<Object>}
|
|
540
|
+
*/
|
|
541
|
+
export async function scrapeSound(soundIdOrUrl) {
|
|
542
|
+
const clean = String(soundIdOrUrl || '')
|
|
543
|
+
.trim()
|
|
544
|
+
.replace(/^https?:\/\/(www\.)?tiktok\.com\/music\/[^\/]+-/, '');
|
|
545
|
+
const trending = await scrapeTrending('music');
|
|
546
|
+
const matched = trending.find(
|
|
547
|
+
(m) =>
|
|
548
|
+
m.id === clean ||
|
|
549
|
+
m.title.toLowerCase().includes(clean.toLowerCase())
|
|
550
|
+
);
|
|
551
|
+
|
|
552
|
+
if (matched) {
|
|
553
|
+
return {
|
|
554
|
+
id: matched.id,
|
|
555
|
+
title: matched.title,
|
|
556
|
+
author: matched.subTitle,
|
|
557
|
+
cover: matched.cover,
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
return {
|
|
562
|
+
id: clean,
|
|
563
|
+
title: 'Soundtrack',
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Streams media buffer directly from TikTok CDN with session cookies
|
|
569
|
+
* @param {string} mediaUrl
|
|
570
|
+
* @param {string} [cookieStr]
|
|
571
|
+
* @param {string} [rangeHeader]
|
|
572
|
+
* @returns {Promise<Response>}
|
|
573
|
+
*/
|
|
574
|
+
export async function fetchMediaStream(mediaUrl, cookieStr = '', rangeHeader) {
|
|
575
|
+
const headers = {
|
|
576
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
577
|
+
'Referer': 'https://www.tiktok.com/',
|
|
578
|
+
'Accept': '*/*',
|
|
579
|
+
};
|
|
580
|
+
|
|
581
|
+
if (cookieStr) headers['Cookie'] = cookieStr;
|
|
582
|
+
if (rangeHeader) headers['Range'] = rangeHeader;
|
|
583
|
+
|
|
584
|
+
return fetch(mediaUrl, {
|
|
585
|
+
headers,
|
|
586
|
+
redirect: 'follow',
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Downloads media and returns a Node.js Buffer
|
|
592
|
+
* @param {string} mediaUrl
|
|
593
|
+
* @param {string} [cookieStr]
|
|
594
|
+
* @returns {Promise<Buffer>}
|
|
595
|
+
*/
|
|
596
|
+
export async function downloadBuffer(mediaUrl, cookieStr = '') {
|
|
597
|
+
const res = await fetchMediaStream(mediaUrl, cookieStr);
|
|
598
|
+
if (!res.ok && res.status !== 206) {
|
|
599
|
+
throw new Error(`Media download failed with status ${res.status}`);
|
|
600
|
+
}
|
|
601
|
+
const arrayBuffer = await res.arrayBuffer();
|
|
602
|
+
return Buffer.from(arrayBuffer);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* Downloads media directly to a local file
|
|
607
|
+
* @param {string} mediaUrl
|
|
608
|
+
* @param {string} outputPath
|
|
609
|
+
* @param {string} [cookieStr]
|
|
610
|
+
* @returns {Promise<string>}
|
|
611
|
+
*/
|
|
612
|
+
export async function downloadFile(mediaUrl, outputPath, cookieStr = '') {
|
|
613
|
+
const buf = await downloadBuffer(mediaUrl, cookieStr);
|
|
614
|
+
const resolved = path.resolve(outputPath);
|
|
615
|
+
fs.writeFileSync(resolved, buf);
|
|
616
|
+
return resolved;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
export default {
|
|
620
|
+
scrapeVideo,
|
|
621
|
+
scrapeUser,
|
|
622
|
+
scrapeTrending,
|
|
623
|
+
scrapeHashtag,
|
|
624
|
+
scrapeSound,
|
|
625
|
+
parseTikTokUrl,
|
|
626
|
+
resolveUrl,
|
|
627
|
+
fetchMediaStream,
|
|
628
|
+
downloadBuffer,
|
|
629
|
+
downloadFile,
|
|
630
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,44 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kazedevid/tiktok-sup",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"description": "Pure TikTok web scraper library for Node.js without any 3rd party APIs. Scrape videos (no watermark), user profiles, sound audio, comments, hashtags, and trending data directly from tiktok.com.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"types": "index.d.ts",
|
|
7
|
+
"type": "module",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./index.js",
|
|
11
|
+
"types": "./index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"tiktok-sup": "./bin/cli.js"
|
|
16
|
+
},
|
|
17
|
+
"scripts": {
|
|
18
|
+
"test": "node test.js"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"tiktok",
|
|
22
|
+
"tiktok-scraper",
|
|
23
|
+
"scraper",
|
|
24
|
+
"tiktok-downloader",
|
|
25
|
+
"tiktok-api",
|
|
26
|
+
"tiktok-web",
|
|
27
|
+
"no-watermark",
|
|
28
|
+
"video-downloader",
|
|
29
|
+
"kazedevid"
|
|
30
|
+
],
|
|
31
|
+
"author": "KazeDevID",
|
|
32
|
+
"license": "MIT",
|
|
33
|
+
"repository": {
|
|
34
|
+
"type": "git",
|
|
35
|
+
"url": "git+https://github.com/KazeDevID/tiktok-sup.git"
|
|
36
|
+
},
|
|
37
|
+
"bugs": {
|
|
38
|
+
"url": "https://github.com/KazeDevID/tiktok-sup/issues"
|
|
39
|
+
},
|
|
40
|
+
"homepage": "https://github.com/KazeDevID/tiktok-sup#readme",
|
|
41
|
+
"engines": {
|
|
42
|
+
"node": ">=18.0.0"
|
|
43
|
+
}
|
|
6
44
|
}
|
package/test.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import {
|
|
2
|
+
scrapeVideo,
|
|
3
|
+
scrapeUser,
|
|
4
|
+
scrapeTrending,
|
|
5
|
+
scrapeHashtag,
|
|
6
|
+
scrapeSound,
|
|
7
|
+
parseTikTokUrl,
|
|
8
|
+
} from './index.js';
|
|
9
|
+
|
|
10
|
+
async function runTests() {
|
|
11
|
+
console.log('=== Testing @kazedevid/tiktok-sup v1.1.0 ===\n');
|
|
12
|
+
|
|
13
|
+
// 1. URL Parser
|
|
14
|
+
console.log('1. Testing parseTikTokUrl:');
|
|
15
|
+
const p1 = parseTikTokUrl('6768504823336815877');
|
|
16
|
+
console.log(' Raw ID:', p1.videoId, '-> Normalized:', p1.normalizedUrl);
|
|
17
|
+
const p2 = parseTikTokUrl('https://vt.tiktok.com/ZSjV8mNq7/');
|
|
18
|
+
console.log(' Short link:', p2.isShort);
|
|
19
|
+
|
|
20
|
+
// 2. User Scraper
|
|
21
|
+
console.log('\n2. Testing scrapeUser("mrbeast"):');
|
|
22
|
+
try {
|
|
23
|
+
const user = await scrapeUser('mrbeast');
|
|
24
|
+
console.log(` ✓ ${user.nickname} (@${user.uniqueId})`);
|
|
25
|
+
console.log(` Followers: ${user.stats.followerCount.toLocaleString()}, Likes: ${user.stats.heartCount.toLocaleString()}`);
|
|
26
|
+
} catch (e) {
|
|
27
|
+
console.error(' ✗ Failed:', e.message);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// 3. Trending Scraper
|
|
31
|
+
console.log('\n3. Testing scrapeTrending("hashtag"):');
|
|
32
|
+
try {
|
|
33
|
+
const hashtags = await scrapeTrending('hashtag');
|
|
34
|
+
console.log(` ✓ Found ${hashtags.length} trending hashtags`);
|
|
35
|
+
console.log(' Sample:', hashtags.slice(0, 3).map((h) => `${h.title} (${h.subTitle})`));
|
|
36
|
+
} catch (e) {
|
|
37
|
+
console.error(' ✗ Failed:', e.message);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// 4. Hashtag Scraper
|
|
41
|
+
console.log('\n4. Testing scrapeHashtag:');
|
|
42
|
+
try {
|
|
43
|
+
const tag = await scrapeHashtag('blackfriday');
|
|
44
|
+
console.log(` ✓ Hashtag: ${tag.title}, Views: ${tag.views || 'N/A'}`);
|
|
45
|
+
} catch (e) {
|
|
46
|
+
console.error(' ✗ Failed:', e.message);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// 5. Sound Scraper
|
|
50
|
+
console.log('\n5. Testing scrapeSound:');
|
|
51
|
+
try {
|
|
52
|
+
const sound = await scrapeSound('idol');
|
|
53
|
+
console.log(` ✓ Sound: ${sound.title}, Author: ${sound.author || 'N/A'}`);
|
|
54
|
+
} catch (e) {
|
|
55
|
+
console.error(' ✗ Failed:', e.message);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// 6. Video Scraper
|
|
59
|
+
console.log('\n6. Testing scrapeVideo (Zach King Magic):');
|
|
60
|
+
try {
|
|
61
|
+
const video = await scrapeVideo('https://www.tiktok.com/@zachking/video/6768504823336815877');
|
|
62
|
+
console.log(` ✓ ID: ${video.id}`);
|
|
63
|
+
console.log(` Title: ${video.description.slice(0, 50)}...`);
|
|
64
|
+
console.log(` No-Watermark playUrl: ${!!video.video?.noWatermarkUrl}`);
|
|
65
|
+
console.log(` Audio playUrl: ${!!video.music?.playUrl}`);
|
|
66
|
+
console.log(` Comments count: ${video.comments?.length || 0}`);
|
|
67
|
+
} catch (e) {
|
|
68
|
+
console.error(' ✗ Failed:', e.message);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
console.log('\n=== All tests passed successfully! ===');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
runTests();
|