@kazedevid/tiktok-sup 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +145 -2
- package/bin/cli.js +96 -0
- package/index.d.ts +133 -0
- package/index.js +374 -0
- package/package.json +41 -3
- package/test.js +41 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 KazeDevID
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,146 @@
|
|
|
1
|
-
#
|
|
1
|
+
# tiktok-sup 🚀
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **Pure TikTok Web Scraper for Node.js** — Scrapes video metadata, direct MP4 streams, MP3 audio, user profiles, comments, and trending hashtags directly from `tiktok.com`. **100% pure web scraping with zero 3rd-party APIs or proxies.**
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/tiktok-sup)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
[](https://github.com/KazeDevID/tiktok-sup)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## ✨ Features
|
|
12
|
+
|
|
13
|
+
- 🎯 **Pure tiktok.com scraping** — Zero reliance on rapidapi, tikwm, ssstik, or external proxies.
|
|
14
|
+
- ⚡ **Lightning Fast** — Direct lightweight parsing of TikTok mobile SSR rehydration & web data.
|
|
15
|
+
- 🎬 **Video & Photo Slides** — Supports standard video posts, short links (`vt.tiktok.com`, `vm.tiktok.com`), and multi-image photo carousels.
|
|
16
|
+
- 🎵 **Audio / Music Extractor** — Extract sound titles, author names, durations, and direct audio stream URLs.
|
|
17
|
+
- 👤 **Full User Profile Scraper** — Followers, following count, total likes/hearts, video counts, bio, secUid, and verified badges.
|
|
18
|
+
- 💬 **Comments Extractor** — Scrapes initial comment lists directly from the post data.
|
|
19
|
+
- 📈 **Trending Explorer** — Discovers trending creators, hashtags, and viral sounds.
|
|
20
|
+
- 🛡️ **Session Cookie Handling** — Returns session cookies required to stream video and audio directly from TikTok CDN.
|
|
21
|
+
- 📦 **TypeScript Support** — Full type definitions out of the box.
|
|
22
|
+
- 💻 **CLI Included** — Run directly via `npx tiktok-sup` in your terminal.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 📦 Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
npm install tiktok-sup
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
or with yarn / pnpm:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
yarn add tiktok-sup
|
|
36
|
+
# or
|
|
37
|
+
pnpm add tiktok-sup
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## 🚀 Quick Start
|
|
43
|
+
|
|
44
|
+
### 1. Scrape Video Metadata & Stream Links
|
|
45
|
+
|
|
46
|
+
```javascript
|
|
47
|
+
import { scrapeVideo } from 'tiktok-sup';
|
|
48
|
+
|
|
49
|
+
// Works with any TikTok URL (desktop, mobile, or vt.tiktok.com short links)
|
|
50
|
+
const video = await scrapeVideo('https://www.tiktok.com/@zachking/video/6768504823336815877');
|
|
51
|
+
|
|
52
|
+
console.log('Title/Desc:', video.description);
|
|
53
|
+
console.log('Author:', video.author.nickname, `(@${video.author.username})`);
|
|
54
|
+
console.log('Stats:', video.stats); // { views, likes, comments, shares, bookmarks }
|
|
55
|
+
console.log('Video Play URL:', video.video?.playUrl);
|
|
56
|
+
console.log('Music:', video.music?.title, '-', video.music?.playUrl);
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### 2. Scrape User Profile
|
|
60
|
+
|
|
61
|
+
```javascript
|
|
62
|
+
import { scrapeUser } from 'tiktok-sup';
|
|
63
|
+
|
|
64
|
+
const profile = await scrapeUser('mrbeast');
|
|
65
|
+
|
|
66
|
+
console.log('Name:', profile.nickname);
|
|
67
|
+
console.log('Followers:', profile.stats.followerCount.toLocaleString());
|
|
68
|
+
console.log('Total Likes:', profile.stats.heartCount.toLocaleString());
|
|
69
|
+
console.log('Bio:', profile.signature);
|
|
70
|
+
console.log('Verified:', profile.verified);
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### 3. Explore Trending Creators & Hashtags
|
|
74
|
+
|
|
75
|
+
```javascript
|
|
76
|
+
import { scrapeTrending } from 'tiktok-sup';
|
|
77
|
+
|
|
78
|
+
const trending = await scrapeTrending();
|
|
79
|
+
trending.forEach(item => {
|
|
80
|
+
console.log(`[${item.type}] ${item.title} - ${item.subTitle || ''}`);
|
|
81
|
+
});
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 4. Downloading or Streaming Videos Directly
|
|
85
|
+
|
|
86
|
+
TikTok CDN requires session cookies and matching referer headers to stream:
|
|
87
|
+
|
|
88
|
+
```javascript
|
|
89
|
+
import { scrapeVideo, fetchMediaStream } from 'tiktok-sup';
|
|
90
|
+
import fs from 'fs';
|
|
91
|
+
|
|
92
|
+
const video = await scrapeVideo('https://vt.tiktok.com/...');
|
|
93
|
+
const streamUrl = video.video.downloadUrl || video.video.playUrl;
|
|
94
|
+
|
|
95
|
+
// fetchMediaStream automatically attaches TikTok referer and session cookies
|
|
96
|
+
const response = await fetchMediaStream(streamUrl, video.sessionCookies);
|
|
97
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
98
|
+
|
|
99
|
+
fs.writeFileSync('video.mp4', buffer);
|
|
100
|
+
console.log('Downloaded video.mp4 successfully!');
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 💻 CLI Usage
|
|
106
|
+
|
|
107
|
+
You can use `tiktok-sup` directly in your terminal without writing code:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
# Scrape video info
|
|
111
|
+
npx tiktok-sup video "https://www.tiktok.com/@zachking/video/6768504823336815877"
|
|
112
|
+
|
|
113
|
+
# Scrape user profile
|
|
114
|
+
npx tiktok-sup user khaby.lame
|
|
115
|
+
|
|
116
|
+
# List trending topics
|
|
117
|
+
npx tiktok-sup trending
|
|
118
|
+
|
|
119
|
+
# Download video MP4 directly
|
|
120
|
+
npx tiktok-sup download "https://vt.tiktok.com/..." output.mp4
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 📚 API Reference
|
|
126
|
+
|
|
127
|
+
### `scrapeVideo(url: string): Promise<TikTokVideoResult>`
|
|
128
|
+
Parses full video metadata, author info, statistics, audio URLs, direct stream URLs, and comments.
|
|
129
|
+
|
|
130
|
+
### `scrapeUser(username: string): Promise<TikTokUserProfile>`
|
|
131
|
+
Scrapes a user profile by username handle (with or without `@`).
|
|
132
|
+
|
|
133
|
+
### `scrapeTrending(): Promise<TikTokTrendingItem[]>`
|
|
134
|
+
Fetches trending creators, hashtags, and sound tracks from `tiktok.com/node/share/discover`.
|
|
135
|
+
|
|
136
|
+
### `parseTikTokUrl(url: string)`
|
|
137
|
+
Extracts videoId and username from any format.
|
|
138
|
+
|
|
139
|
+
### `fetchMediaStream(url: string, cookieStr?: string): Promise<Response>`
|
|
140
|
+
Fetches the raw media buffer with TikTok headers.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## 📄 License
|
|
145
|
+
|
|
146
|
+
MIT © [KazeDevID](https://github.com/KazeDevID)
|
package/bin/cli.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* tiktok-sup CLI
|
|
5
|
+
* Command line tool for scraping TikTok without 3rd party APIs
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { scrapeVideo, scrapeUser, scrapeTrending } from '../index.js';
|
|
9
|
+
import fs from 'fs';
|
|
10
|
+
import path from 'path';
|
|
11
|
+
|
|
12
|
+
const args = process.argv.slice(2);
|
|
13
|
+
const command = args[0];
|
|
14
|
+
const target = args[1];
|
|
15
|
+
|
|
16
|
+
async function main() {
|
|
17
|
+
if (!command || command === '--help' || command === '-h') {
|
|
18
|
+
console.log(`
|
|
19
|
+
tiktok-sup - Pure TikTok Web Scraper for Node.js (tiktok.com only)
|
|
20
|
+
|
|
21
|
+
Usage:
|
|
22
|
+
npx tiktok-sup video <url> Scrape full video metadata, audio & stream links
|
|
23
|
+
npx tiktok-sup user <username> Scrape user profile and statistics
|
|
24
|
+
npx tiktok-sup trending List trending creators, hashtags and music
|
|
25
|
+
npx tiktok-sup download <url> [out] Download video MP4 to file
|
|
26
|
+
|
|
27
|
+
Examples:
|
|
28
|
+
npx tiktok-sup video https://www.tiktok.com/@zachking/video/6768504823336815877
|
|
29
|
+
npx tiktok-sup user khaby.lame
|
|
30
|
+
npx tiktok-sup download https://vt.tiktok.com/ZS... ./myvideo.mp4
|
|
31
|
+
`);
|
|
32
|
+
process.exit(0);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
try {
|
|
36
|
+
if (command === 'video') {
|
|
37
|
+
if (!target) {
|
|
38
|
+
console.error('Error: Please provide a TikTok video URL.');
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
console.log(`Scraping video from ${target} directly from tiktok.com...`);
|
|
42
|
+
const data = await scrapeVideo(target);
|
|
43
|
+
console.log(JSON.stringify(data, null, 2));
|
|
44
|
+
} else if (command === 'user') {
|
|
45
|
+
if (!target) {
|
|
46
|
+
console.error('Error: Please provide a TikTok username.');
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
console.log(`Scraping profile @${target} directly from tiktok.com...`);
|
|
50
|
+
const data = await scrapeUser(target);
|
|
51
|
+
console.log(JSON.stringify(data, null, 2));
|
|
52
|
+
} else if (command === 'trending') {
|
|
53
|
+
console.log('Fetching trending creators, hashtags and music from tiktok.com...');
|
|
54
|
+
const data = await scrapeTrending();
|
|
55
|
+
console.log(JSON.stringify(data, null, 2));
|
|
56
|
+
} else if (command === 'download') {
|
|
57
|
+
if (!target) {
|
|
58
|
+
console.error('Error: Please provide a TikTok video URL.');
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
61
|
+
console.log(`Resolving stream URL for ${target}...`);
|
|
62
|
+
const data = await scrapeVideo(target);
|
|
63
|
+
const streamUrl = data.video?.downloadUrl || data.video?.playUrl;
|
|
64
|
+
if (!streamUrl) {
|
|
65
|
+
console.error('No video stream URL found for this post.');
|
|
66
|
+
process.exit(1);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const outFile = args[2] || `${data.id || 'tiktok_video'}.mp4`;
|
|
70
|
+
console.log(`Downloading stream to ${outFile}...`);
|
|
71
|
+
const res = await fetch(streamUrl, {
|
|
72
|
+
headers: {
|
|
73
|
+
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15',
|
|
74
|
+
'Referer': 'https://www.tiktok.com/',
|
|
75
|
+
...(data.sessionCookies ? { Cookie: data.sessionCookies } : {}),
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
if (!res.ok) {
|
|
80
|
+
throw new Error(`Download failed: HTTP ${res.status}`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const buffer = Buffer.from(await res.arrayBuffer());
|
|
84
|
+
fs.writeFileSync(path.resolve(outFile), buffer);
|
|
85
|
+
console.log(`Saved ${buffer.length} bytes to ${outFile}!`);
|
|
86
|
+
} else {
|
|
87
|
+
console.error(`Unknown command: "${command}". Run "tiktok-sup --help" for instructions.`);
|
|
88
|
+
process.exit(1);
|
|
89
|
+
}
|
|
90
|
+
} catch (err) {
|
|
91
|
+
console.error(`Error: ${err.message}`);
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
main();
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
export interface TikTokAuthor {
|
|
2
|
+
id: string;
|
|
3
|
+
username: string;
|
|
4
|
+
nickname: string;
|
|
5
|
+
avatar: string;
|
|
6
|
+
verified: boolean;
|
|
7
|
+
secUid?: string;
|
|
8
|
+
signature?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface TikTokStats {
|
|
12
|
+
views: number;
|
|
13
|
+
likes: number;
|
|
14
|
+
comments: number;
|
|
15
|
+
shares: number;
|
|
16
|
+
bookmarks: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface TikTokVideoMedia {
|
|
20
|
+
id: string;
|
|
21
|
+
duration: number;
|
|
22
|
+
width: number;
|
|
23
|
+
height: number;
|
|
24
|
+
ratio?: string;
|
|
25
|
+
cover: string;
|
|
26
|
+
dynamicCover?: string;
|
|
27
|
+
playUrl?: string;
|
|
28
|
+
downloadUrl?: string;
|
|
29
|
+
format?: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface TikTokMusic {
|
|
33
|
+
id: string;
|
|
34
|
+
title: string;
|
|
35
|
+
author: string;
|
|
36
|
+
duration: number;
|
|
37
|
+
playUrl?: string;
|
|
38
|
+
cover?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface TikTokComment {
|
|
42
|
+
cid: string;
|
|
43
|
+
text: string;
|
|
44
|
+
createTime: number;
|
|
45
|
+
diggCount: number;
|
|
46
|
+
replyCount: number;
|
|
47
|
+
user: {
|
|
48
|
+
id: string;
|
|
49
|
+
username: string;
|
|
50
|
+
nickname: string;
|
|
51
|
+
avatar: string;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface TikTokVideoResult {
|
|
56
|
+
id: string;
|
|
57
|
+
url: string;
|
|
58
|
+
description: string;
|
|
59
|
+
createTime: number;
|
|
60
|
+
createTimeFormatted: string;
|
|
61
|
+
author: TikTokAuthor;
|
|
62
|
+
stats: TikTokStats;
|
|
63
|
+
video?: TikTokVideoMedia;
|
|
64
|
+
music?: TikTokMusic;
|
|
65
|
+
isPhoto: boolean;
|
|
66
|
+
photos?: string[];
|
|
67
|
+
comments?: TikTokComment[];
|
|
68
|
+
sessionCookies?: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface TikTokUserProfile {
|
|
72
|
+
id: string;
|
|
73
|
+
uniqueId: string;
|
|
74
|
+
nickname: string;
|
|
75
|
+
avatar: string;
|
|
76
|
+
avatarMedium: string;
|
|
77
|
+
avatarThumb: string;
|
|
78
|
+
signature: string;
|
|
79
|
+
bioLink?: { link: string; risk: number };
|
|
80
|
+
verified: boolean;
|
|
81
|
+
secUid: string;
|
|
82
|
+
privateAccount: boolean;
|
|
83
|
+
stats: {
|
|
84
|
+
followerCount: number;
|
|
85
|
+
followingCount: number;
|
|
86
|
+
heartCount: number;
|
|
87
|
+
videoCount: number;
|
|
88
|
+
friendCount: number;
|
|
89
|
+
};
|
|
90
|
+
shareMeta?: {
|
|
91
|
+
title: string;
|
|
92
|
+
desc: string;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export interface TikTokTrendingItem {
|
|
97
|
+
type: 'creator' | 'hashtag' | 'music' | 'unknown';
|
|
98
|
+
id: string;
|
|
99
|
+
title: string;
|
|
100
|
+
subTitle?: string;
|
|
101
|
+
desc?: string;
|
|
102
|
+
cover?: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export declare function parseTikTokUrl(inputUrl: string): {
|
|
106
|
+
videoId?: string;
|
|
107
|
+
username?: string;
|
|
108
|
+
isShort: boolean;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
export declare function resolveUrl(inputUrl: string): Promise<{
|
|
112
|
+
canonicalUrl: string;
|
|
113
|
+
cookies: string;
|
|
114
|
+
}>;
|
|
115
|
+
|
|
116
|
+
export declare function scrapeVideo(inputUrl: string): Promise<TikTokVideoResult>;
|
|
117
|
+
|
|
118
|
+
export declare function scrapeUser(inputUsername: string): Promise<TikTokUserProfile>;
|
|
119
|
+
|
|
120
|
+
export declare function scrapeTrending(): Promise<TikTokTrendingItem[]>;
|
|
121
|
+
|
|
122
|
+
export declare function fetchMediaStream(mediaUrl: string, cookieStr?: string): Promise<Response>;
|
|
123
|
+
|
|
124
|
+
declare const _default: {
|
|
125
|
+
scrapeVideo: typeof scrapeVideo;
|
|
126
|
+
scrapeUser: typeof scrapeUser;
|
|
127
|
+
scrapeTrending: typeof scrapeTrending;
|
|
128
|
+
parseTikTokUrl: typeof parseTikTokUrl;
|
|
129
|
+
resolveUrl: typeof resolveUrl;
|
|
130
|
+
fetchMediaStream: typeof fetchMediaStream;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export default _default;
|
package/index.js
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tiktok-sup
|
|
3
|
+
* Pure TikTok web scraper for Node.js without any 3rd party APIs.
|
|
4
|
+
* Scrapes videos, users, audio, comments & trending data directly from tiktok.com.
|
|
5
|
+
*
|
|
6
|
+
* @author KazeDevID
|
|
7
|
+
* @license MIT
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const MOBILE_USER_AGENT =
|
|
11
|
+
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1';
|
|
12
|
+
|
|
13
|
+
const DESKTOP_USER_AGENT =
|
|
14
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Parses and normalizes any TikTok URL format
|
|
18
|
+
* @param {string} inputUrl
|
|
19
|
+
* @returns {{ videoId?: string, username?: string, isShort: boolean }}
|
|
20
|
+
*/
|
|
21
|
+
export function parseTikTokUrl(inputUrl) {
|
|
22
|
+
const clean = String(inputUrl || '').trim();
|
|
23
|
+
const isShort = clean.includes('vt.tiktok.com') || clean.includes('vm.tiktok.com') || clean.includes('/t/');
|
|
24
|
+
const videoMatch = clean.match(/\/video\/(\d{15,22})/);
|
|
25
|
+
const photoMatch = clean.match(/\/photo\/(\d{15,22})/);
|
|
26
|
+
const userMatch = clean.match(/@([a-zA-Z0-9_.-]+)/);
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
videoId: videoMatch?.[1] || photoMatch?.[1],
|
|
30
|
+
username: userMatch?.[1],
|
|
31
|
+
isShort,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Resolves short links (vt.tiktok.com, vm.tiktok.com) to the final canonical URL and returns cookies
|
|
37
|
+
* @param {string} inputUrl
|
|
38
|
+
* @returns {Promise<{ canonicalUrl: string, cookies: string }>}
|
|
39
|
+
*/
|
|
40
|
+
export async function resolveUrl(inputUrl) {
|
|
41
|
+
let url = String(inputUrl || '').trim();
|
|
42
|
+
if (!url.startsWith('http')) {
|
|
43
|
+
url = 'https://' + url;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const res = await fetch(url, {
|
|
47
|
+
headers: {
|
|
48
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
49
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
50
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
51
|
+
},
|
|
52
|
+
redirect: 'follow',
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
const cookiesList = (res.headers.getSetCookie ? res.headers.getSetCookie() : [res.headers.get('set-cookie')]).filter(Boolean);
|
|
56
|
+
const cookies = cookiesList.map((c) => c.split(';')[0]).join('; ');
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
canonicalUrl: res.url,
|
|
60
|
+
cookies,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Scrapes a TikTok video or photo post directly from tiktok.com
|
|
66
|
+
* @param {string} inputUrl
|
|
67
|
+
* @returns {Promise<Object>}
|
|
68
|
+
*/
|
|
69
|
+
export async function scrapeVideo(inputUrl) {
|
|
70
|
+
const { canonicalUrl, cookies: initialCookies } = await resolveUrl(inputUrl);
|
|
71
|
+
|
|
72
|
+
const res = await fetch(canonicalUrl, {
|
|
73
|
+
headers: {
|
|
74
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
75
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
76
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
77
|
+
'Referer': 'https://www.tiktok.com/',
|
|
78
|
+
...(initialCookies ? { Cookie: initialCookies } : {}),
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
if (!res.ok && res.status !== 200) {
|
|
83
|
+
throw new Error(`Failed to fetch video page from tiktok.com: HTTP ${res.status}`);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const pageCookiesList = (res.headers.getSetCookie ? res.headers.getSetCookie() : [res.headers.get('set-cookie')]).filter(Boolean);
|
|
87
|
+
const pageCookies = pageCookiesList.map((c) => c.split(';')[0]).join('; ');
|
|
88
|
+
const combinedCookies = [initialCookies, pageCookies].filter(Boolean).join('; ');
|
|
89
|
+
|
|
90
|
+
const html = await res.text();
|
|
91
|
+
|
|
92
|
+
let itemStruct = null;
|
|
93
|
+
let commentsRaw = [];
|
|
94
|
+
|
|
95
|
+
// 1. Try parsing from <script id="api-data">
|
|
96
|
+
const apiDataMatch = html.match(/<script id="api-data"[^>]*>([\s\S]*?)<\/script>/);
|
|
97
|
+
if (apiDataMatch) {
|
|
98
|
+
try {
|
|
99
|
+
const parsed = JSON.parse(apiDataMatch[1]);
|
|
100
|
+
if (parsed.videoDetail?.itemInfo?.itemStruct) {
|
|
101
|
+
itemStruct = parsed.videoDetail.itemInfo.itemStruct;
|
|
102
|
+
}
|
|
103
|
+
if (parsed.commentList?.comments && Array.isArray(parsed.commentList.comments)) {
|
|
104
|
+
commentsRaw = parsed.commentList.comments;
|
|
105
|
+
}
|
|
106
|
+
} catch (_) {}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// 2. Fallback to <script id="__UNIVERSAL_DATA_FOR_REHYDRATION__">
|
|
110
|
+
if (!itemStruct) {
|
|
111
|
+
const rehydrationMatch = html.match(/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/);
|
|
112
|
+
if (rehydrationMatch) {
|
|
113
|
+
try {
|
|
114
|
+
const parsed = JSON.parse(rehydrationMatch[1]);
|
|
115
|
+
const scope = parsed['__DEFAULT_SCOPE__'] || {};
|
|
116
|
+
const videoDetail = scope['webapp.video-detail'];
|
|
117
|
+
if (videoDetail?.itemInfo?.itemStruct) {
|
|
118
|
+
itemStruct = videoDetail.itemInfo.itemStruct;
|
|
119
|
+
}
|
|
120
|
+
} catch (_) {}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// 3. Fallback to SIGI_STATE
|
|
125
|
+
if (!itemStruct) {
|
|
126
|
+
const sigiMatch = html.match(/<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/);
|
|
127
|
+
if (sigiMatch) {
|
|
128
|
+
try {
|
|
129
|
+
const parsed = JSON.parse(sigiMatch[1]);
|
|
130
|
+
const items = parsed.ItemModule || {};
|
|
131
|
+
const firstKey = Object.keys(items)[0];
|
|
132
|
+
if (firstKey && items[firstKey]) {
|
|
133
|
+
itemStruct = items[firstKey];
|
|
134
|
+
}
|
|
135
|
+
} catch (_) {}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (!itemStruct) {
|
|
140
|
+
throw new Error('Could not locate TikTok video data on tiktok.com. Video may be private, region-restricted, or deleted.');
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Parse photos for slideshow posts
|
|
144
|
+
const photos = [];
|
|
145
|
+
if (itemStruct.imagePost?.images && Array.isArray(itemStruct.imagePost.images)) {
|
|
146
|
+
for (const img of itemStruct.imagePost.images) {
|
|
147
|
+
const u = img.displayImage?.urlList?.[0] || img.thumbnail?.urlList?.[0] || img.imageURL?.urlList?.[0];
|
|
148
|
+
if (u) photos.push(u);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const author = {
|
|
153
|
+
id: String(itemStruct.author?.id || ''),
|
|
154
|
+
username: String(itemStruct.author?.uniqueId || ''),
|
|
155
|
+
nickname: String(itemStruct.author?.nickname || ''),
|
|
156
|
+
avatar: String(itemStruct.author?.avatarLarger || itemStruct.author?.avatarMedium || itemStruct.author?.avatarThumb || ''),
|
|
157
|
+
verified: Boolean(itemStruct.author?.verified),
|
|
158
|
+
secUid: itemStruct.author?.secUid,
|
|
159
|
+
signature: itemStruct.author?.signature,
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const stats = {
|
|
163
|
+
views: Number(itemStruct.stats?.playCount || 0),
|
|
164
|
+
likes: Number(itemStruct.stats?.diggCount || 0),
|
|
165
|
+
comments: Number(itemStruct.stats?.commentCount || 0),
|
|
166
|
+
shares: Number(itemStruct.stats?.shareCount || 0),
|
|
167
|
+
bookmarks: Number(itemStruct.stats?.collectCount || 0),
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
let video = undefined;
|
|
171
|
+
if (itemStruct.video) {
|
|
172
|
+
video = {
|
|
173
|
+
id: String(itemStruct.video.id || itemStruct.id || ''),
|
|
174
|
+
duration: Number(itemStruct.video.duration || 0),
|
|
175
|
+
width: Number(itemStruct.video.width || 0),
|
|
176
|
+
height: Number(itemStruct.video.height || 0),
|
|
177
|
+
ratio: itemStruct.video.ratio,
|
|
178
|
+
cover: String(itemStruct.video.cover || ''),
|
|
179
|
+
dynamicCover: itemStruct.video.dynamicCover ? String(itemStruct.video.dynamicCover) : undefined,
|
|
180
|
+
playUrl: itemStruct.video.playAddr ? String(itemStruct.video.playAddr) : undefined,
|
|
181
|
+
downloadUrl: itemStruct.video.downloadAddr ? String(itemStruct.video.downloadAddr) : itemStruct.video.playAddr ? String(itemStruct.video.playAddr) : undefined,
|
|
182
|
+
format: itemStruct.video.format,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let music = undefined;
|
|
187
|
+
if (itemStruct.music) {
|
|
188
|
+
music = {
|
|
189
|
+
id: String(itemStruct.music.id || ''),
|
|
190
|
+
title: String(itemStruct.music.title || 'Original Sound'),
|
|
191
|
+
author: String(itemStruct.music.authorName || author.nickname),
|
|
192
|
+
duration: Number(itemStruct.music.duration || 0),
|
|
193
|
+
playUrl: itemStruct.music.playUrl ? String(itemStruct.music.playUrl) : undefined,
|
|
194
|
+
cover: itemStruct.music.coverLarge ? String(itemStruct.music.coverLarge) : itemStruct.music.coverMedium ? String(itemStruct.music.coverMedium) : undefined,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const comments = commentsRaw.map((c) => ({
|
|
199
|
+
cid: String(c.cid || ''),
|
|
200
|
+
text: String(c.text || ''),
|
|
201
|
+
createTime: Number(c.create_time || 0),
|
|
202
|
+
diggCount: Number(c.digg_count || 0),
|
|
203
|
+
replyCount: Number(c.reply_comment_total || 0),
|
|
204
|
+
user: {
|
|
205
|
+
id: String(c.user?.uid || ''),
|
|
206
|
+
username: String(c.user?.unique_id || ''),
|
|
207
|
+
nickname: String(c.user?.nickname || ''),
|
|
208
|
+
avatar: String(c.user?.avatar_thumb?.url_list?.[0] || ''),
|
|
209
|
+
},
|
|
210
|
+
}));
|
|
211
|
+
|
|
212
|
+
const createTime = Number(itemStruct.createTime || 0);
|
|
213
|
+
const createDate = createTime > 0 ? new Date(createTime * 1000) : new Date();
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
id: String(itemStruct.id || ''),
|
|
217
|
+
url: canonicalUrl,
|
|
218
|
+
description: String(itemStruct.desc || ''),
|
|
219
|
+
createTime,
|
|
220
|
+
createTimeFormatted: createDate.toLocaleString('en-US', {
|
|
221
|
+
year: 'numeric',
|
|
222
|
+
month: 'short',
|
|
223
|
+
day: 'numeric',
|
|
224
|
+
hour: '2-digit',
|
|
225
|
+
minute: '2-digit',
|
|
226
|
+
}),
|
|
227
|
+
author,
|
|
228
|
+
stats,
|
|
229
|
+
video,
|
|
230
|
+
music,
|
|
231
|
+
isPhoto: photos.length > 0,
|
|
232
|
+
photos: photos.length > 0 ? photos : undefined,
|
|
233
|
+
comments: comments.length > 0 ? comments : undefined,
|
|
234
|
+
sessionCookies: combinedCookies,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Scrapes a TikTok user profile directly from tiktok.com
|
|
240
|
+
* @param {string} inputUsername
|
|
241
|
+
* @returns {Promise<Object>}
|
|
242
|
+
*/
|
|
243
|
+
export async function scrapeUser(inputUsername) {
|
|
244
|
+
const username = String(inputUsername || '').trim().replace(/^@+/, '');
|
|
245
|
+
if (!username) {
|
|
246
|
+
throw new Error('Please provide a valid TikTok username (e.g. "khaby.lame" or "@mrbeast").');
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const url = `https://www.tiktok.com/@${encodeURIComponent(username)}`;
|
|
250
|
+
const res = await fetch(url, {
|
|
251
|
+
headers: {
|
|
252
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
253
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
254
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
255
|
+
},
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
if (!res.ok) {
|
|
259
|
+
throw new Error(`Failed to fetch user @${username}: HTTP ${res.status}`);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const html = await res.text();
|
|
263
|
+
const match = html.match(/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/);
|
|
264
|
+
|
|
265
|
+
if (!match) {
|
|
266
|
+
throw new Error(`Could not parse user @${username} data from tiktok.com.`);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const data = JSON.parse(match[1]);
|
|
270
|
+
const userDetail = data['__DEFAULT_SCOPE__']?.['webapp.user-detail'];
|
|
271
|
+
if (!userDetail?.userInfo?.user) {
|
|
272
|
+
throw new Error(`TikTok user @${username} does not exist or is unavailable.`);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
const u = userDetail.userInfo.user;
|
|
276
|
+
const s = userDetail.userInfo.stats || {};
|
|
277
|
+
|
|
278
|
+
return {
|
|
279
|
+
id: String(u.id || ''),
|
|
280
|
+
uniqueId: String(u.uniqueId || username),
|
|
281
|
+
nickname: String(u.nickname || username),
|
|
282
|
+
avatar: String(u.avatarLarger || u.avatarMedium || u.avatarThumb || ''),
|
|
283
|
+
avatarMedium: String(u.avatarMedium || u.avatarThumb || ''),
|
|
284
|
+
avatarThumb: String(u.avatarThumb || ''),
|
|
285
|
+
signature: String(u.signature || ''),
|
|
286
|
+
bioLink: u.bioLink,
|
|
287
|
+
verified: Boolean(u.verified),
|
|
288
|
+
secUid: String(u.secUid || ''),
|
|
289
|
+
privateAccount: Boolean(u.privateAccount),
|
|
290
|
+
stats: {
|
|
291
|
+
followerCount: Number(s.followerCount || 0),
|
|
292
|
+
followingCount: Number(s.followingCount || 0),
|
|
293
|
+
heartCount: Number(s.heartCount || s.heart || 0),
|
|
294
|
+
videoCount: Number(s.videoCount || 0),
|
|
295
|
+
friendCount: Number(s.friendCount || 0),
|
|
296
|
+
},
|
|
297
|
+
shareMeta: userDetail.shareMeta,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Scrapes trending creators, hashtags, and sound tracks directly from tiktok.com
|
|
303
|
+
* @returns {Promise<Array<Object>>}
|
|
304
|
+
*/
|
|
305
|
+
export async function scrapeTrending() {
|
|
306
|
+
const res = await fetch('https://www.tiktok.com/node/share/discover', {
|
|
307
|
+
headers: {
|
|
308
|
+
'User-Agent': DESKTOP_USER_AGENT,
|
|
309
|
+
'Accept': 'application/json, text/plain, */*',
|
|
310
|
+
'Referer': 'https://www.tiktok.com/',
|
|
311
|
+
},
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
if (!res.ok) {
|
|
315
|
+
throw new Error(`Failed to fetch trending discovery data: HTTP ${res.status}`);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const json = await res.json();
|
|
319
|
+
const results = [];
|
|
320
|
+
|
|
321
|
+
if (Array.isArray(json.body)) {
|
|
322
|
+
for (const group of json.body) {
|
|
323
|
+
if (Array.isArray(group.exploreList)) {
|
|
324
|
+
for (const item of group.exploreList) {
|
|
325
|
+
const card = item.cardItem;
|
|
326
|
+
if (!card) continue;
|
|
327
|
+
|
|
328
|
+
let type = 'unknown';
|
|
329
|
+
if (card.type === 2) type = 'creator';
|
|
330
|
+
else if (card.type === 3) type = 'hashtag';
|
|
331
|
+
else if (card.type === 1) type = 'music';
|
|
332
|
+
|
|
333
|
+
results.push({
|
|
334
|
+
type,
|
|
335
|
+
id: String(card.id || ''),
|
|
336
|
+
title: String(card.title || ''),
|
|
337
|
+
subTitle: card.subTitle ? String(card.subTitle) : undefined,
|
|
338
|
+
desc: card.desc ? String(card.desc) : undefined,
|
|
339
|
+
cover: card.cover ? String(card.cover) : undefined,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return results;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Streams or downloads media buffer directly from TikTok CDN with session cookies
|
|
351
|
+
* @param {string} mediaUrl
|
|
352
|
+
* @param {string} [cookieStr]
|
|
353
|
+
* @returns {Promise<Response>}
|
|
354
|
+
*/
|
|
355
|
+
export async function fetchMediaStream(mediaUrl, cookieStr = '') {
|
|
356
|
+
return fetch(mediaUrl, {
|
|
357
|
+
headers: {
|
|
358
|
+
'User-Agent': MOBILE_USER_AGENT,
|
|
359
|
+
'Referer': 'https://www.tiktok.com/',
|
|
360
|
+
'Accept': '*/*',
|
|
361
|
+
...(cookieStr ? { Cookie: cookieStr } : {}),
|
|
362
|
+
},
|
|
363
|
+
redirect: 'follow',
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
export default {
|
|
368
|
+
scrapeVideo,
|
|
369
|
+
scrapeUser,
|
|
370
|
+
scrapeTrending,
|
|
371
|
+
parseTikTokUrl,
|
|
372
|
+
resolveUrl,
|
|
373
|
+
fetchMediaStream,
|
|
374
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,44 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kazedevid/tiktok-sup",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Pure TikTok web scraper library for Node.js without any 3rd party APIs. Scrape videos, user profiles, sound audio, comments and trending data directly from tiktok.com.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"types": "index.d.ts",
|
|
7
|
+
"type": "module",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./index.js",
|
|
11
|
+
"types": "./index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"tiktok-sup": "./bin/cli.js"
|
|
16
|
+
},
|
|
17
|
+
"scripts": {
|
|
18
|
+
"test": "node test.js"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"tiktok",
|
|
22
|
+
"tiktok-scraper",
|
|
23
|
+
"scraper",
|
|
24
|
+
"tiktok-downloader",
|
|
25
|
+
"tiktok-api",
|
|
26
|
+
"tiktok-web",
|
|
27
|
+
"no-watermark",
|
|
28
|
+
"video-downloader",
|
|
29
|
+
"kazedevid"
|
|
30
|
+
],
|
|
31
|
+
"author": "KazeDevID",
|
|
32
|
+
"license": "MIT",
|
|
33
|
+
"repository": {
|
|
34
|
+
"type": "git",
|
|
35
|
+
"url": "git+https://github.com/KazeDevID/tiktok-sup.git"
|
|
36
|
+
},
|
|
37
|
+
"bugs": {
|
|
38
|
+
"url": "https://github.com/KazeDevID/tiktok-sup/issues"
|
|
39
|
+
},
|
|
40
|
+
"homepage": "https://github.com/KazeDevID/tiktok-sup#readme",
|
|
41
|
+
"engines": {
|
|
42
|
+
"node": ">=18.0.0"
|
|
43
|
+
}
|
|
6
44
|
}
|
package/test.js
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { scrapeVideo, scrapeUser, scrapeTrending } from './index.js';
|
|
2
|
+
|
|
3
|
+
async function runTests() {
|
|
4
|
+
console.log('--- Testing tiktok-sup library ---');
|
|
5
|
+
|
|
6
|
+
// Test 1: scrapeUser
|
|
7
|
+
console.log('\n1. Testing scrapeUser("mrbeast")...');
|
|
8
|
+
try {
|
|
9
|
+
const user = await scrapeUser('mrbeast');
|
|
10
|
+
console.log('✓ Success:', user.nickname, `(${user.uniqueId})`, 'Followers:', user.stats.followerCount);
|
|
11
|
+
} catch (e) {
|
|
12
|
+
console.error('✗ Failed scrapeUser:', e.message);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// Test 2: scrapeTrending
|
|
16
|
+
console.log('\n2. Testing scrapeTrending()...');
|
|
17
|
+
try {
|
|
18
|
+
const trending = await scrapeTrending();
|
|
19
|
+
console.log('✓ Success: Retrieved', trending.length, 'trending items');
|
|
20
|
+
console.log('Sample:', trending.slice(0, 3).map(t => `${t.title} (${t.type})`));
|
|
21
|
+
} catch (e) {
|
|
22
|
+
console.error('✗ Failed scrapeTrending:', e.message);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Test 3: scrapeVideo
|
|
26
|
+
console.log('\n3. Testing scrapeVideo("https://www.tiktok.com/@zachking/video/6768504823336815877")...');
|
|
27
|
+
try {
|
|
28
|
+
const video = await scrapeVideo('https://www.tiktok.com/@zachking/video/6768504823336815877');
|
|
29
|
+
console.log('✓ Success:', video.description.slice(0, 50) + '...');
|
|
30
|
+
console.log('Author:', video.author.nickname);
|
|
31
|
+
console.log('Likes:', video.stats.likes, 'Comments count:', video.stats.comments);
|
|
32
|
+
console.log('Video playUrl present:', !!video.video?.playUrl);
|
|
33
|
+
console.log('Music playUrl present:', !!video.music?.playUrl);
|
|
34
|
+
} catch (e) {
|
|
35
|
+
console.error('✗ Failed scrapeVideo:', e.message);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
console.log('\n--- All tests completed ---');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
runTests();
|