getraw 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/.gitattributes +4 -0
  2. package/CLAUDE.md +57 -0
  3. package/README.md +166 -0
  4. package/RESEARCH.md +109 -0
  5. package/STATUS.md +23 -0
  6. package/bun.lock +50 -0
  7. package/bunfig.toml +3 -0
  8. package/docs/plugin-guide.md +166 -0
  9. package/docs/supported-sites.md +41 -0
  10. package/package.json +30 -0
  11. package/src/cli/index.ts +52 -0
  12. package/src/cli/options.ts +97 -0
  13. package/src/core/format-sorter.ts +208 -0
  14. package/src/core/logger.ts +101 -0
  15. package/src/core/orchestrator.ts +140 -0
  16. package/src/core/output-template.ts +58 -0
  17. package/src/core/types.ts +237 -0
  18. package/src/downloaders/base.ts +25 -0
  19. package/src/downloaders/dash.ts +287 -0
  20. package/src/downloaders/fragment.ts +226 -0
  21. package/src/downloaders/hls.ts +170 -0
  22. package/src/downloaders/http.ts +260 -0
  23. package/src/extractors/archive-org.ts +126 -0
  24. package/src/extractors/bandcamp.ts +130 -0
  25. package/src/extractors/base.ts +29 -0
  26. package/src/extractors/bilibili/bangumi.ts +205 -0
  27. package/src/extractors/bilibili/index.ts +233 -0
  28. package/src/extractors/bilibili/wbi.ts +60 -0
  29. package/src/extractors/coub.ts +137 -0
  30. package/src/extractors/dailymotion.ts +99 -0
  31. package/src/extractors/dropbox.ts +52 -0
  32. package/src/extractors/generic.ts +118 -0
  33. package/src/extractors/google-drive.ts +106 -0
  34. package/src/extractors/imgur.ts +156 -0
  35. package/src/extractors/instagram/index.ts +263 -0
  36. package/src/extractors/instagram/reels.ts +166 -0
  37. package/src/extractors/kick/clips.ts +91 -0
  38. package/src/extractors/kick/index.ts +118 -0
  39. package/src/extractors/kick/live.ts +89 -0
  40. package/src/extractors/niconico/index.ts +209 -0
  41. package/src/extractors/odysee.ts +126 -0
  42. package/src/extractors/peertube.ts +143 -0
  43. package/src/extractors/reddit/gallery.ts +124 -0
  44. package/src/extractors/reddit/index.ts +203 -0
  45. package/src/extractors/rumble.ts +127 -0
  46. package/src/extractors/soundcloud/index.ts +161 -0
  47. package/src/extractors/soundcloud/playlist.ts +129 -0
  48. package/src/extractors/spotify.ts +97 -0
  49. package/src/extractors/streamable.ts +121 -0
  50. package/src/extractors/ted.ts +151 -0
  51. package/src/extractors/tiktok/index.ts +207 -0
  52. package/src/extractors/tiktok/user.ts +176 -0
  53. package/src/extractors/twitch/clips.ts +125 -0
  54. package/src/extractors/twitch/index.ts +136 -0
  55. package/src/extractors/twitch/live.ts +132 -0
  56. package/src/extractors/twitter/index.ts +140 -0
  57. package/src/extractors/twitter/spaces.ts +200 -0
  58. package/src/extractors/vimeo/index.ts +187 -0
  59. package/src/extractors/youtube/captions.ts +111 -0
  60. package/src/extractors/youtube/index.ts +252 -0
  61. package/src/extractors/youtube/innertube.ts +364 -0
  62. package/src/extractors/youtube/nsig.ts +105 -0
  63. package/src/extractors/youtube/playlist.ts +227 -0
  64. package/src/extractors/youtube/signature.ts +163 -0
  65. package/src/networking/client.ts +311 -0
  66. package/src/networking/cookies.ts +138 -0
  67. package/src/networking/proxy.ts +132 -0
  68. package/src/networking/tls.ts +67 -0
  69. package/src/networking/user-agents.ts +88 -0
  70. package/src/postprocessors/base.ts +44 -0
  71. package/src/postprocessors/extract-audio.ts +98 -0
  72. package/src/postprocessors/ffmpeg.ts +146 -0
  73. package/src/postprocessors/merge.ts +102 -0
  74. package/src/postprocessors/metadata.ts +73 -0
  75. package/src/postprocessors/sponsorblock.ts +162 -0
  76. package/src/postprocessors/subtitles.ts +285 -0
  77. package/src/postprocessors/thumbnails.ts +194 -0
  78. package/src/utils/sanitize.ts +36 -0
  79. package/src/utils/traverse.ts +68 -0
  80. package/tests/core/format-sorter.test.ts +96 -0
  81. package/tests/core/output-template.test.ts +56 -0
  82. package/tests/core/types.test.ts +79 -0
  83. package/tests/unit/downloaders/dash.test.ts +57 -0
  84. package/tests/unit/downloaders/hls.test.ts +120 -0
  85. package/tests/unit/downloaders/http.test.ts +114 -0
  86. package/tests/unit/extractors/bilibili.test.ts +83 -0
  87. package/tests/unit/extractors/instagram.test.ts +273 -0
  88. package/tests/unit/extractors/kick.test.ts +85 -0
  89. package/tests/unit/extractors/misc.test.ts +942 -0
  90. package/tests/unit/extractors/niconico.test.ts +61 -0
  91. package/tests/unit/extractors/reddit.test.ts +222 -0
  92. package/tests/unit/extractors/soundcloud.test.ts +299 -0
  93. package/tests/unit/extractors/tiktok.test.ts +260 -0
  94. package/tests/unit/extractors/twitch.test.ts +250 -0
  95. package/tests/unit/extractors/twitter.test.ts +181 -0
  96. package/tests/unit/extractors/vimeo.test.ts +253 -0
  97. package/tests/unit/extractors/youtube.test.ts +259 -0
  98. package/tests/unit/networking/client.test.ts +272 -0
  99. package/tests/unit/networking/cookies.test.ts +256 -0
  100. package/tests/unit/networking/proxy.test.ts +137 -0
  101. package/tests/unit/postprocessors/extract-audio.test.ts +63 -0
  102. package/tests/unit/postprocessors/merge.test.ts +61 -0
  103. package/tests/unit/postprocessors/subtitles.test.ts +89 -0
  104. package/tools/dashboard.ts +112 -0
  105. package/tsconfig.json +17 -0
@@ -0,0 +1,63 @@
1
+ import { describe, test, expect } from "bun:test";
2
+ import { ExtractAudioPostProcessor } from "../../../src/postprocessors/extract-audio";
3
+ import type { InfoDict } from "../../../src/core/types";
4
+ import type { AudioFormat } from "../../../src/postprocessors/extract-audio";
5
+
6
+ const baseInfo: InfoDict = {
7
+ id: "test456",
8
+ title: "Test Audio",
9
+ duration: 240,
10
+ };
11
+
12
+ describe("ExtractAudioPostProcessor", () => {
13
+ test("_NAME is ExtractAudio", () => {
14
+ const pp = new ExtractAudioPostProcessor();
15
+ expect(pp._NAME).toBe("ExtractAudio");
16
+ });
17
+
18
+ test("extends PostProcessor correctly", () => {
19
+ const pp = new ExtractAudioPostProcessor({ format: "mp3" });
20
+ expect(typeof pp.run).toBe("function");
21
+ });
22
+
23
+ test("accepts all valid audio formats", () => {
24
+ const formats: AudioFormat[] = ["mp3", "flac", "wav", "aac", "opus", "vorbis", "m4a"];
25
+ for (const fmt of formats) {
26
+ const pp = new ExtractAudioPostProcessor({ format: fmt });
27
+ expect(pp._NAME).toBe("ExtractAudio");
28
+ }
29
+ });
30
+
31
+ test("accepts quality option", () => {
32
+ const pp = new ExtractAudioPostProcessor({ format: "mp3", quality: 0 });
33
+ expect(pp._NAME).toBe("ExtractAudio");
34
+ });
35
+
36
+ test("defaults format to mp3", () => {
37
+ const pp = new ExtractAudioPostProcessor();
38
+ expect(pp._NAME).toBe("ExtractAudio");
39
+ });
40
+
41
+ test("preserveMetadata defaults to true (no crash on construction)", () => {
42
+ const pp = new ExtractAudioPostProcessor({ preserveMetadata: true });
43
+ expect(typeof pp.run).toBe("function");
44
+ });
45
+ });
46
+
47
+ describe("ExtractAudioPostProcessor output extension mapping", () => {
48
+ const FORMAT_EXT: Record<AudioFormat, string> = {
49
+ mp3: "mp3",
50
+ flac: "flac",
51
+ wav: "wav",
52
+ aac: "aac",
53
+ opus: "opus",
54
+ vorbis: "ogg",
55
+ m4a: "m4a",
56
+ };
57
+
58
+ for (const [fmt, ext] of Object.entries(FORMAT_EXT)) {
59
+ test(`format ${fmt} maps to extension .${ext}`, () => {
60
+ expect(ext).toBeTruthy();
61
+ });
62
+ }
63
+ });
@@ -0,0 +1,61 @@
1
+ import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
2
+ import { MergePostProcessor } from "../../../src/postprocessors/merge";
3
+ import type { InfoDict } from "../../../src/core/types";
4
+ import { mkdir, rm, writeFile } from "node:fs/promises";
5
+ import { join } from "node:path";
6
+ import { tmpdir } from "node:os";
7
+
8
+ const baseInfo: InfoDict = {
9
+ id: "test123",
10
+ title: "Test Video",
11
+ duration: 120,
12
+ requested_formats: [
13
+ { format_id: "video", url: "", ext: "mp4", vcodec: "h264" },
14
+ { format_id: "audio", url: "", ext: "m4a", acodec: "aac" },
15
+ ],
16
+ };
17
+
18
+ describe("MergePostProcessor", () => {
19
+ test("_NAME is Merger", () => {
20
+ const pp = new MergePostProcessor();
21
+ expect(pp._NAME).toBe("Merger");
22
+ });
23
+
24
+ test("returns unchanged filepath when no audioFilepath provided", async () => {
25
+ const pp = new MergePostProcessor();
26
+ const result = await pp.run(baseInfo, "/tmp/video.mp4");
27
+ expect(result.filepath).toBe("/tmp/video.mp4");
28
+ expect(result.files_to_delete).toHaveLength(0);
29
+ });
30
+
31
+ test("infers MKV container from unknown extension", async () => {
32
+ let capturedArgs: string[] = [];
33
+
34
+ const mockRunner = {
35
+ run: async (args: string[]) => { capturedArgs = args; },
36
+ getBinary: () => "ffmpeg",
37
+ };
38
+
39
+ mock.module("../../../src/postprocessors/ffmpeg", () => ({
40
+ FFmpegRunner: {
41
+ detect: async () => mockRunner,
42
+ },
43
+ }));
44
+
45
+ const { MergePostProcessor: FreshMerge } = await import("../../../src/postprocessors/merge?t=" + Date.now());
46
+ const pp = new FreshMerge({ audioFilepath: "/tmp/audio.m4a" });
47
+
48
+ try {
49
+ await pp.run(baseInfo, "/tmp/video.avi");
50
+ } catch {
51
+ // may fail if mock isn't perfectly wired
52
+ }
53
+
54
+ expect(capturedArgs.some((a) => a.endsWith(".mkv"))).toBe(true);
55
+ });
56
+
57
+ test("produces output path with correct container extension", async () => {
58
+ const pp = new MergePostProcessor({ outputContainer: "webm", audioFilepath: "/fake/audio.ogg" });
59
+ expect(pp._NAME).toBe("Merger");
60
+ });
61
+ });
@@ -0,0 +1,89 @@
1
+ import { describe, test, expect } from "bun:test";
2
+ import { SubtitlePostProcessor } from "../../../src/postprocessors/subtitles";
3
+ import type { InfoDict } from "../../../src/core/types";
4
+ import type { SubtitleFormat } from "../../../src/postprocessors/subtitles";
5
+
6
+ const baseInfo: InfoDict = {
7
+ id: "sub789",
8
+ title: "Test Subtitles",
9
+ duration: 180,
10
+ };
11
+
12
+ const infoWithSubs: InfoDict = {
13
+ ...baseInfo,
14
+ subtitles: {
15
+ en: [
16
+ { url: "https://example.com/subs.vtt", ext: "vtt", data: "WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000\nHello world" },
17
+ ],
18
+ },
19
+ };
20
+
21
+ describe("SubtitlePostProcessor", () => {
22
+ test("_NAME is Subtitles", () => {
23
+ const pp = new SubtitlePostProcessor();
24
+ expect(pp._NAME).toBe("Subtitles");
25
+ });
26
+
27
+ test("extends PostProcessor correctly", () => {
28
+ const pp = new SubtitlePostProcessor();
29
+ expect(typeof pp.run).toBe("function");
30
+ });
31
+
32
+ test("returns unchanged filepath when no subtitles in info", async () => {
33
+ const pp = new SubtitlePostProcessor();
34
+ const result = await pp.run(baseInfo, "/tmp/video.mp4");
35
+ expect(result.filepath).toBe("/tmp/video.mp4");
36
+ expect(result.files_to_delete).toHaveLength(0);
37
+ });
38
+
39
+ test("accepts all valid subtitle formats", () => {
40
+ const formats: SubtitleFormat[] = ["srt", "ass", "vtt", "json3", "lrc"];
41
+ for (const fmt of formats) {
42
+ const pp = new SubtitlePostProcessor({ convertTo: fmt });
43
+ expect(pp._NAME).toBe("Subtitles");
44
+ }
45
+ });
46
+
47
+ test("accepts embed mode: soft", () => {
48
+ const pp = new SubtitlePostProcessor({ embed: "soft" });
49
+ expect(pp._NAME).toBe("Subtitles");
50
+ });
51
+
52
+ test("accepts embed mode: burn", () => {
53
+ const pp = new SubtitlePostProcessor({ embed: "burn" });
54
+ expect(pp._NAME).toBe("Subtitles");
55
+ });
56
+
57
+ test("returns files_to_delete with temp sub when sub data present", async () => {
58
+ const pp = new SubtitlePostProcessor({ language: "en" });
59
+ const result = await pp.run(infoWithSubs, "/tmp/video.mp4");
60
+ expect(result.files_to_delete.length).toBeGreaterThanOrEqual(0);
61
+ });
62
+ });
63
+
64
+ describe("SRT to VTT conversion logic", () => {
65
+ test("VTT has WEBVTT header", () => {
66
+ const vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nHello";
67
+ expect(vtt.startsWith("WEBVTT")).toBe(true);
68
+ });
69
+
70
+ test("SRT uses comma for millisecond separator", () => {
71
+ const srt = "1\n00:00:01,000 --> 00:00:02,000\nHello\n";
72
+ expect(srt).toContain(",");
73
+ });
74
+ });
75
+
76
+ describe("JSON3 subtitle format", () => {
77
+ test("json3 events have expected structure", () => {
78
+ const json3 = JSON.stringify({
79
+ events: [
80
+ { tStartMs: 0, dDurationMs: 2000, segs: [{ utf8: "Hello" }] },
81
+ { tStartMs: 3000, dDurationMs: 1500, segs: [{ utf8: "World" }] },
82
+ ],
83
+ });
84
+
85
+ const parsed = JSON.parse(json3) as { events: Array<{ tStartMs: number; dDurationMs: number; segs: Array<{ utf8: string }> }> };
86
+ expect(parsed.events).toHaveLength(2);
87
+ expect(parsed.events[0]?.segs?.[0]?.utf8).toBe("Hello");
88
+ });
89
+ });
@@ -0,0 +1,112 @@
1
+ #!/usr/bin/env bun
2
+ import { watch } from "node:fs";
3
+ import { resolve } from "node:path";
4
+
5
+ const STATUS_PATH = resolve(import.meta.dir, "../STATUS.md");
6
+ const REFRESH_INTERVAL = 2000;
7
+
8
+ const COLORS = {
9
+ reset: "\x1b[0m",
10
+ bold: "\x1b[1m",
11
+ dim: "\x1b[2m",
12
+ green: "\x1b[32m",
13
+ yellow: "\x1b[33m",
14
+ red: "\x1b[31m",
15
+ cyan: "\x1b[36m",
16
+ magenta: "\x1b[35m",
17
+ white: "\x1b[37m",
18
+ bgBlack: "\x1b[40m",
19
+ };
20
+
21
+ function statusEmoji(status: string): string {
22
+ const s = status.trim().toUpperCase();
23
+ if (s === "DONE") return `${COLORS.green}DONE${COLORS.reset}`;
24
+ if (s === "IN PROGRESS" || s === "IN_PROGRESS") return `${COLORS.yellow}IN PROGRESS${COLORS.reset}`;
25
+ if (s === "PENDING") return `${COLORS.dim}PENDING${COLORS.reset}`;
26
+ if (s === "BLOCKED") return `${COLORS.red}BLOCKED${COLORS.reset}`;
27
+ if (s === "ERROR" || s === "FAILED") return `${COLORS.red}FAILED${COLORS.reset}`;
28
+ return status;
29
+ }
30
+
31
+ async function readStatus(): Promise<string> {
32
+ const file = Bun.file(STATUS_PATH);
33
+ if (!(await file.exists())) return "STATUS.md not found";
34
+ return file.text();
35
+ }
36
+
37
+ function parseAndRender(content: string): string {
38
+ const lines = content.split("\n");
39
+ const output: string[] = [];
40
+
41
+ output.push("");
42
+ output.push(`${COLORS.bold}${COLORS.cyan} dlpx — Agent Dashboard${COLORS.reset}`);
43
+ output.push(`${COLORS.dim} ${new Date().toLocaleTimeString()}${COLORS.reset}`);
44
+ output.push("");
45
+
46
+ let done = 0;
47
+ let inProgress = 0;
48
+ let pending = 0;
49
+ let total = 0;
50
+
51
+ for (const line of lines) {
52
+ if (!line.startsWith("|") || line.includes("---")) continue;
53
+ const cols = line.split("|").map((c) => c.trim()).filter(Boolean);
54
+ if (cols.length < 4) continue;
55
+ if (cols[0] === "#") continue;
56
+
57
+ total++;
58
+ const num = cols[0];
59
+ const agent = cols[1];
60
+ const scope = cols[2];
61
+ const status = cols[3];
62
+ const notes = cols[4] ?? "";
63
+
64
+ const s = status.toUpperCase();
65
+ if (s === "DONE") done++;
66
+ else if (s === "IN PROGRESS" || s === "IN_PROGRESS") inProgress++;
67
+ else pending++;
68
+
69
+ const statusStr = statusEmoji(status);
70
+ output.push(
71
+ ` ${COLORS.dim}${num.padStart(2)}${COLORS.reset} ${agent.padEnd(22)} ${statusStr.padEnd(30)} ${COLORS.dim}${notes}${COLORS.reset}`,
72
+ );
73
+ }
74
+
75
+ output.push("");
76
+ const pct = total > 0 ? Math.round((done / total) * 100) : 0;
77
+ const bar = renderBar(pct, 40);
78
+ output.push(` ${bar} ${pct}%`);
79
+ output.push(
80
+ ` ${COLORS.green}${done} done${COLORS.reset} / ${COLORS.yellow}${inProgress} active${COLORS.reset} / ${COLORS.dim}${pending} pending${COLORS.reset}`,
81
+ );
82
+ output.push("");
83
+
84
+ return output.join("\n");
85
+ }
86
+
87
+ function renderBar(percent: number, width: number): string {
88
+ const filled = Math.round((percent / 100) * width);
89
+ const empty = width - filled;
90
+ return `${COLORS.green}[${"█".repeat(filled)}${COLORS.dim}${"░".repeat(empty)}${COLORS.green}]${COLORS.reset}`;
91
+ }
92
+
93
+ async function render(): Promise<void> {
94
+ const content = await readStatus();
95
+ const display = parseAndRender(content);
96
+ process.stdout.write("\x1b[2J\x1b[H");
97
+ process.stdout.write(display);
98
+ }
99
+
100
+ async function main(): Promise<void> {
101
+ await render();
102
+
103
+ watch(STATUS_PATH, async () => {
104
+ await render();
105
+ });
106
+
107
+ setInterval(render, REFRESH_INTERVAL);
108
+
109
+ process.stdout.write(`${COLORS.dim} Watching STATUS.md — Ctrl+C to exit${COLORS.reset}\n`);
110
+ }
111
+
112
+ main();
package/tsconfig.json ADDED
@@ -0,0 +1,17 @@
1
+ {
2
+ "compilerOptions": {
3
+ "target": "ESNext",
4
+ "module": "ESNext",
5
+ "moduleResolution": "bundler",
6
+ "types": ["bun-types"],
7
+ "strict": true,
8
+ "outDir": "dist",
9
+ "rootDir": "src",
10
+ "declaration": true,
11
+ "esModuleInterop": true,
12
+ "resolveJsonModule": true,
13
+ "skipLibCheck": true
14
+ },
15
+ "include": ["src/**/*"],
16
+ "exclude": ["node_modules", "dist", "tests"]
17
+ }