ai-weekly 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +194 -0
- package/bin/ai-weekly.mjs +54 -0
- package/package.json +34 -0
- package/scripts/build-report.mjs +98 -0
- package/scripts/download-report-covers.mjs +145 -0
- package/scripts/fetch-subtitles.mjs +335 -0
- package/scripts/review-creators.mjs +170 -0
- package/scripts/run-discovery.mjs +375 -0
- package/scripts/run-weekly.mjs +88 -0
- package/scripts/summarize-videos.mjs +400 -0
- package/scripts/synthesize-projects.mjs +246 -0
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { mkdtemp, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
|
+
import { pathToFileURL } from "node:url";
|
|
6
|
+
|
|
7
|
+
function usage() {
|
|
8
|
+
console.log(`
|
|
9
|
+
用法:
|
|
10
|
+
npx ai-weekly summarize [选项]
|
|
11
|
+
|
|
12
|
+
只读取已经保存的字幕工件,生成可审核的结构化视频摘要;不下载音频,也不执行转写。
|
|
13
|
+
|
|
14
|
+
选项:
|
|
15
|
+
--date YYYY-MM-DD 读取和写入 data/runs/<日期>/;默认今天
|
|
16
|
+
--input PATH subtitles.json 路径;默认 data/runs/<日期>/subtitles.json
|
|
17
|
+
--output-dir PATH 摘要输出目录;默认与输入文件同级的 summaries/
|
|
18
|
+
--model MODEL Codex 使用的模型;默认 gpt-5.6-luna
|
|
19
|
+
--limit N 最多摘要 N 条视频;默认全部
|
|
20
|
+
--all 明确摘要字幕索引中的全部视频(与默认相同)
|
|
21
|
+
--force 即使已有有效摘要也重新调用 Codex
|
|
22
|
+
--timeout-ms N 单条 Codex 摘要超时毫秒数;默认 180000
|
|
23
|
+
--concurrency N 同时运行的 Codex 摘要数;默认 1,最多 2
|
|
24
|
+
--help 显示本帮助
|
|
25
|
+
`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function parseArgs(argv) {
|
|
29
|
+
const options = {
|
|
30
|
+
date: new Date().toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" }),
|
|
31
|
+
input: null,
|
|
32
|
+
outputDir: null,
|
|
33
|
+
model: "gpt-5.6-luna",
|
|
34
|
+
limit: Infinity,
|
|
35
|
+
timeoutMs: 180000,
|
|
36
|
+
concurrency: 1,
|
|
37
|
+
force: false
|
|
38
|
+
};
|
|
39
|
+
const valueOptions = new Map([["--date", "date"], ["--input", "input"], ["--output-dir", "outputDir"], ["--model", "model"], ["--limit", "limit"], ["--timeout-ms", "timeoutMs"], ["--concurrency", "concurrency"]]);
|
|
40
|
+
|
|
41
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
42
|
+
const arg = argv[index];
|
|
43
|
+
if (arg === "--help") return { help: true };
|
|
44
|
+
if (arg === "--all") { options.limit = Infinity; continue; }
|
|
45
|
+
if (arg === "--force") { options.force = true; continue; }
|
|
46
|
+
const key = valueOptions.get(arg);
|
|
47
|
+
if (!key || index + 1 >= argv.length) throw new Error(`无效参数:${arg}`);
|
|
48
|
+
options[key] = ["limit", "timeoutMs", "concurrency"].includes(key) ? Number(argv[index + 1]) : argv[index + 1];
|
|
49
|
+
index += 1;
|
|
50
|
+
}
|
|
51
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(options.date)) throw new Error("--date 必须为 YYYY-MM-DD");
|
|
52
|
+
if ((!Number.isInteger(options.limit) || options.limit < 1) && options.limit !== Infinity) throw new Error("--limit 必须是正整数");
|
|
53
|
+
if (!Number.isInteger(options.timeoutMs) || options.timeoutMs < 1000) throw new Error("--timeout-ms 必须是不少于 1000 的整数");
|
|
54
|
+
if (!Number.isInteger(options.concurrency) || options.concurrency < 1 || options.concurrency > 2) throw new Error("--concurrency 必须是 1 或 2");
|
|
55
|
+
return options;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function writeJsonAtomically(path, value) {
|
|
59
|
+
const temporaryPath = `${path}.tmp-${process.pid}`;
|
|
60
|
+
await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`);
|
|
61
|
+
await rename(temporaryPath, path);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function isPathWithin(directory, path) {
|
|
65
|
+
return path === directory || path.startsWith(`${directory}${sep}`);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function resolveSavedSubtitle(inputDirectory, file) {
|
|
69
|
+
if (typeof file !== "string" || file.length === 0 || isAbsolute(file)) {
|
|
70
|
+
throw new Error("字幕工件路径必须是相对路径");
|
|
71
|
+
}
|
|
72
|
+
const path = resolve(inputDirectory, file);
|
|
73
|
+
if (!isPathWithin(inputDirectory, path)) throw new Error("字幕工件路径不能离开字幕索引目录");
|
|
74
|
+
return path;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function subtitleText(subtitle) {
|
|
78
|
+
if (!Array.isArray(subtitle?.body)) throw new Error("字幕工件缺少 subtitle.body");
|
|
79
|
+
const text = subtitle.body.map((line) => String(line?.content || "").trim()).filter(Boolean).join("\n");
|
|
80
|
+
if (!text) throw new Error("字幕工件不包含可摘要的文字");
|
|
81
|
+
return text;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const reportSchema = {
|
|
85
|
+
type: "object",
|
|
86
|
+
additionalProperties: false,
|
|
87
|
+
required: ["project_facts", "video_viewpoints"],
|
|
88
|
+
properties: {
|
|
89
|
+
project_facts: { type: "string" },
|
|
90
|
+
video_viewpoints: { type: "string" }
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
const summarySchema = {
|
|
95
|
+
type: "object",
|
|
96
|
+
additionalProperties: false,
|
|
97
|
+
required: ["facts", "viewpoints", "demonstrations_or_usage", "limitations", "report"],
|
|
98
|
+
properties: {
|
|
99
|
+
...Object.fromEntries(["facts", "viewpoints", "demonstrations_or_usage", "limitations"].map((category) => [category, {
|
|
100
|
+
type: "array",
|
|
101
|
+
items: { type: "string" },
|
|
102
|
+
maxItems: 4
|
|
103
|
+
}])),
|
|
104
|
+
report: reportSchema
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
function validateSummary(value) {
|
|
109
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Codex 返回的摘要不是 JSON 对象");
|
|
110
|
+
for (const category of ["facts", "viewpoints", "demonstrations_or_usage", "limitations"]) {
|
|
111
|
+
if (!Array.isArray(value[category]) || value[category].some((item) => typeof item !== "string" || !item.trim())) {
|
|
112
|
+
throw new Error(`Codex 返回的 ${category} 不符合摘要结构`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const summary = Object.fromEntries(["facts", "viewpoints", "demonstrations_or_usage", "limitations"].map((category) => [category, value[category].map((item) => item.trim())]));
|
|
116
|
+
const report = value.report;
|
|
117
|
+
if (!report || typeof report !== "object" || Array.isArray(report)) throw new Error("Codex 返回的周报摘要不符合结构");
|
|
118
|
+
for (const key of reportSchema.required) if (typeof report[key] !== "string" || !report[key].trim()) throw new Error(`Codex 返回的 report.${key} 不符合结构`);
|
|
119
|
+
return { ...summary, report: Object.fromEntries(reportSchema.required.map((key) => [key, report[key].trim()])) };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function runCodex(args, prompt, timeoutMs) {
|
|
123
|
+
return new Promise((resolve, reject) => {
|
|
124
|
+
const child = spawn("codex", args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
125
|
+
let stderr = "";
|
|
126
|
+
const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
|
|
127
|
+
child.stdout.resume();
|
|
128
|
+
child.stderr.on("data", (chunk) => { stderr += chunk; });
|
|
129
|
+
child.once("error", (error) => {
|
|
130
|
+
clearTimeout(timer);
|
|
131
|
+
reject(error);
|
|
132
|
+
});
|
|
133
|
+
child.once("close", (code, signal) => {
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
if (code === 0) return resolve(stderr);
|
|
136
|
+
const reason = signal === "SIGTERM" ? `超过 ${timeoutMs}ms 未完成` : `退出码 ${code}`;
|
|
137
|
+
reject(new Error(`${reason}${stderr ? `:${stderr.trim().slice(-500)}` : ""}`));
|
|
138
|
+
});
|
|
139
|
+
child.stdin.end(prompt);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function usageFromCodex(stderr, configuredModel) {
|
|
144
|
+
const tokenMatch = stderr.match(/tokens used\s*\n\s*([\d,]+)/i);
|
|
145
|
+
const modelMatch = stderr.match(/^model:\s*(\S+)/mi);
|
|
146
|
+
return { model: modelMatch?.[1] || configuredModel || "default", tokens: tokenMatch ? Number(tokenMatch[1].replaceAll(",", "")) : null };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export async function summarizeWithCodex({ subtitleText: text, repository, videoTitle, model, timeoutMs }) {
|
|
150
|
+
const directory = await mkdtemp(join(tmpdir(), "ai-weekly-codex-summary-"));
|
|
151
|
+
const schemaPath = join(directory, "summary-schema.json");
|
|
152
|
+
const outputPath = join(directory, "summary.json");
|
|
153
|
+
try {
|
|
154
|
+
await writeFile(schemaPath, `${JSON.stringify(summarySchema)}\n`);
|
|
155
|
+
const args = ["exec", "--skip-git-repo-check", "--ephemeral", "--ignore-user-config", "--sandbox", "read-only", "--output-schema", schemaPath, "--output-last-message", outputPath, "--cd", directory];
|
|
156
|
+
if (model) args.push("--model", model);
|
|
157
|
+
args.push("-");
|
|
158
|
+
const stderr = await runCodex(args, `仅依据下列字幕纯文本,为目标项目 ${repository || "未命名项目"} 的视频《${videoTitle || "未命名视频"}》生成中文结构化摘要。区分视频陈述的事实、观点、演示或使用结论、限制条件;不确定的内容不要编造。\n\n同时填写 report。视频可能一次性介绍多个项目:只提取与目标项目直接相关的信息,跳过其他项目的名称、事实、评价和使用方法;无论是否涉及多个项目,都必须生成 project_facts 和 video_viewpoints 两项。两项各写成信息充分、自然连贯的中文短段,目标为 120 至 180 字;字幕信息不足时可更短,但不得为了凑字数补充材料外的信息。project_facts 只写可确认的项目能力、流程、演示结果和使用方式;video_viewpoints 概括视频给出的评价、适用判断与取舍,不要出现“视频作者”“作者认为”。语气自然克制,不用宣传词。\n\n最终回答必须只符合提供的 JSON Schema,不要解释。\n\n字幕:\n${text}\n`, timeoutMs);
|
|
159
|
+
return { summary: validateSummary(JSON.parse(await readFile(outputPath, "utf8"))), usage: usageFromCodex(stderr, model) };
|
|
160
|
+
} catch (error) {
|
|
161
|
+
throw new Error(`Codex 摘要失败:${error.message}`);
|
|
162
|
+
} finally {
|
|
163
|
+
await rm(directory, { recursive: true, force: true });
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function savedSubtitleEntries(video) {
|
|
168
|
+
return (video.subtitles || []).filter((subtitle) => ["downloaded", "already_downloaded"].includes(subtitle.status));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function pendingStatus(status) {
|
|
172
|
+
const names = {
|
|
173
|
+
login_required: "pending_login_required",
|
|
174
|
+
no_subtitles: "pending_no_subtitles",
|
|
175
|
+
error: "pending_subtitle_error"
|
|
176
|
+
};
|
|
177
|
+
return names[status] || "pending_no_saved_subtitles";
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function summarizedStatus(status) {
|
|
181
|
+
const names = {
|
|
182
|
+
login_required: "summarized_with_pending_login_required",
|
|
183
|
+
no_subtitles: "summarized_with_pending_no_subtitles",
|
|
184
|
+
error: "summarized_with_pending_subtitle_error"
|
|
185
|
+
};
|
|
186
|
+
return names[status] || "summarized";
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function summaryFileName(video, subtitle) {
|
|
190
|
+
const id = String(subtitle.id);
|
|
191
|
+
const language = String(subtitle.language);
|
|
192
|
+
if (!/^[A-Za-z0-9_-]+$/.test(video.bvid || "") || !/^[A-Za-z0-9_-]+$/.test(id) || !/^[A-Za-z0-9_-]+$/.test(language)) {
|
|
193
|
+
throw new Error("视频或字幕标识包含不安全字符");
|
|
194
|
+
}
|
|
195
|
+
return `${video.bvid}-${id}-${language}.json`;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async function readSavedSubtitle(inputDirectory, video, subtitle) {
|
|
199
|
+
const artifactPath = resolveSavedSubtitle(inputDirectory, subtitle.file);
|
|
200
|
+
const artifact = JSON.parse(await readFile(artifactPath, "utf8"));
|
|
201
|
+
if (artifact.schema_version !== "1.0" || artifact.source?.bvid !== video.bvid) {
|
|
202
|
+
throw new Error("字幕工件来源与字幕索引不一致");
|
|
203
|
+
}
|
|
204
|
+
if (String(artifact.source?.subtitle_id) !== String(subtitle.id) || artifact.source?.language !== subtitle.language) {
|
|
205
|
+
throw new Error("字幕工件标识与字幕索引不一致");
|
|
206
|
+
}
|
|
207
|
+
return { artifact, artifactPath };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async function hasValidSummary(path, video, subtitle) {
|
|
211
|
+
try {
|
|
212
|
+
const artifact = JSON.parse(await readFile(path, "utf8"));
|
|
213
|
+
return artifact.schema_version === "1.0"
|
|
214
|
+
&& artifact.source?.bvid === video.bvid
|
|
215
|
+
&& String(artifact.source?.subtitle?.subtitle_id) === String(subtitle.id)
|
|
216
|
+
&& artifact.source?.subtitle?.language === subtitle.language
|
|
217
|
+
&& artifact.summary
|
|
218
|
+
&& (() => { try { validateSummary(artifact.summary); return true; } catch { return false; } })();
|
|
219
|
+
} catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async function mapWithConcurrency(items, concurrency, map) {
|
|
225
|
+
const results = new Array(items.length);
|
|
226
|
+
let nextIndex = 0;
|
|
227
|
+
const worker = async () => {
|
|
228
|
+
while (true) {
|
|
229
|
+
const index = nextIndex;
|
|
230
|
+
nextIndex += 1;
|
|
231
|
+
if (index >= items.length) return;
|
|
232
|
+
results[index] = await map(items[index], index);
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
|
|
236
|
+
return results;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export async function runVideoSummaries(options, { summarize = summarizeWithCodex, log = console.log } = {}) {
|
|
240
|
+
const defaultRunDirectory = resolve("data", "runs", options.date);
|
|
241
|
+
const inputPath = resolve(options.input || join(defaultRunDirectory, "subtitles.json"));
|
|
242
|
+
const inputDirectory = dirname(inputPath);
|
|
243
|
+
const outputDir = resolve(options.outputDir || join(inputDirectory, "summaries"));
|
|
244
|
+
const manifestPath = join(dirname(outputDir), "summaries.json");
|
|
245
|
+
const subtitlesManifest = JSON.parse(await readFile(inputPath, "utf8"));
|
|
246
|
+
if (!Array.isArray(subtitlesManifest.videos)) throw new Error("subtitles.json 缺少 videos");
|
|
247
|
+
const videos = subtitlesManifest.videos.slice(0, options.limit ?? Infinity);
|
|
248
|
+
const timeoutMs = options.timeoutMs ?? 180000;
|
|
249
|
+
|
|
250
|
+
await mkdir(outputDir, { recursive: true });
|
|
251
|
+
const records = await mapWithConcurrency(videos, options.concurrency ?? 1, async (video, index) => {
|
|
252
|
+
const availableSubtitles = savedSubtitleEntries(video);
|
|
253
|
+
if (availableSubtitles.length === 0) {
|
|
254
|
+
const record = {
|
|
255
|
+
repository: video.repository,
|
|
256
|
+
project_rank: video.project_rank,
|
|
257
|
+
bvid: video.bvid,
|
|
258
|
+
video_url: video.video_url,
|
|
259
|
+
video_title: video.video_title,
|
|
260
|
+
creator: video.creator,
|
|
261
|
+
creator_id: video.creator_id,
|
|
262
|
+
creator_url: video.creator_url,
|
|
263
|
+
status: pendingStatus(video.status),
|
|
264
|
+
subtitle_status: video.status,
|
|
265
|
+
error: video.error
|
|
266
|
+
};
|
|
267
|
+
log(`[${index + 1}/${videos.length}] ${video.bvid}: ${pendingStatus(video.status)}`);
|
|
268
|
+
return record;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
const subtitle = availableSubtitles[0];
|
|
272
|
+
try {
|
|
273
|
+
const { artifact, artifactPath } = await readSavedSubtitle(inputDirectory, video, subtitle);
|
|
274
|
+
const fileName = summaryFileName(video, subtitle);
|
|
275
|
+
const outputPath = join(outputDir, fileName);
|
|
276
|
+
if (!options.force && await hasValidSummary(outputPath, video, subtitle)) {
|
|
277
|
+
const status = `already_${summarizedStatus(video.status)}`;
|
|
278
|
+
const record = {
|
|
279
|
+
repository: video.repository,
|
|
280
|
+
project_rank: video.project_rank,
|
|
281
|
+
bvid: video.bvid,
|
|
282
|
+
video_url: video.video_url,
|
|
283
|
+
video_title: video.video_title,
|
|
284
|
+
creator: video.creator,
|
|
285
|
+
creator_id: video.creator_id,
|
|
286
|
+
creator_url: video.creator_url,
|
|
287
|
+
status,
|
|
288
|
+
pending_subtitle_status: ["login_required", "no_subtitles", "error"].includes(video.status) ? video.status : undefined,
|
|
289
|
+
error: video.error,
|
|
290
|
+
summary_file: relative(dirname(manifestPath), outputPath),
|
|
291
|
+
source_subtitle: { file: relative(dirname(manifestPath), artifactPath), source_url: artifact.source.subtitle_url }
|
|
292
|
+
};
|
|
293
|
+
log(`[${index + 1}/${videos.length}] ${video.bvid}: ${status}`);
|
|
294
|
+
return record;
|
|
295
|
+
}
|
|
296
|
+
log(`[${index + 1}/${videos.length}] ${video.bvid}: 正在请求 Codex…`);
|
|
297
|
+
const result = await summarize({ subtitleText: subtitleText(artifact.subtitle), repository: video.repository, videoTitle: video.video_title, model: options.model, timeoutMs });
|
|
298
|
+
const summary = result.summary || result;
|
|
299
|
+
const usage = result.usage || { model: options.model || "default", tokens: null };
|
|
300
|
+
await writeJsonAtomically(outputPath, {
|
|
301
|
+
schema_version: "1.0",
|
|
302
|
+
source: {
|
|
303
|
+
platform: artifact.source.platform,
|
|
304
|
+
bvid: artifact.source.bvid,
|
|
305
|
+
cid: artifact.source.cid,
|
|
306
|
+
video_url: artifact.source.video_url,
|
|
307
|
+
video_title: video.video_title,
|
|
308
|
+
creator: video.creator,
|
|
309
|
+
creator_id: video.creator_id,
|
|
310
|
+
creator_url: video.creator_url,
|
|
311
|
+
subtitle: {
|
|
312
|
+
file: relative(outputDir, artifactPath),
|
|
313
|
+
subtitle_id: artifact.source.subtitle_id,
|
|
314
|
+
language: artifact.source.language,
|
|
315
|
+
source_url: artifact.source.subtitle_url,
|
|
316
|
+
fetched_at: artifact.source.fetched_at
|
|
317
|
+
}
|
|
318
|
+
},
|
|
319
|
+
summarizer: { provider: "codex_cli", ...usage },
|
|
320
|
+
summary
|
|
321
|
+
});
|
|
322
|
+
const record = {
|
|
323
|
+
repository: video.repository,
|
|
324
|
+
project_rank: video.project_rank,
|
|
325
|
+
bvid: video.bvid,
|
|
326
|
+
video_url: video.video_url,
|
|
327
|
+
video_title: video.video_title,
|
|
328
|
+
creator: video.creator,
|
|
329
|
+
creator_id: video.creator_id,
|
|
330
|
+
creator_url: video.creator_url,
|
|
331
|
+
status: summarizedStatus(video.status),
|
|
332
|
+
pending_subtitle_status: ["login_required", "no_subtitles", "error"].includes(video.status) ? video.status : undefined,
|
|
333
|
+
error: video.error,
|
|
334
|
+
summary_file: relative(dirname(manifestPath), outputPath),
|
|
335
|
+
source_subtitle: {
|
|
336
|
+
file: relative(dirname(manifestPath), artifactPath),
|
|
337
|
+
source_url: artifact.source.subtitle_url
|
|
338
|
+
},
|
|
339
|
+
usage
|
|
340
|
+
};
|
|
341
|
+
log(`[${index + 1}/${videos.length}] ${video.bvid}: summarized`);
|
|
342
|
+
return record;
|
|
343
|
+
} catch (error) {
|
|
344
|
+
const record = {
|
|
345
|
+
repository: video.repository,
|
|
346
|
+
project_rank: video.project_rank,
|
|
347
|
+
bvid: video.bvid,
|
|
348
|
+
video_url: video.video_url,
|
|
349
|
+
video_title: video.video_title,
|
|
350
|
+
creator: video.creator,
|
|
351
|
+
creator_id: video.creator_id,
|
|
352
|
+
creator_url: video.creator_url,
|
|
353
|
+
status: "error",
|
|
354
|
+
error: error.message
|
|
355
|
+
};
|
|
356
|
+
log(`[${index + 1}/${videos.length}] ${video.bvid}: error (${error.message})`);
|
|
357
|
+
return record;
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
const summary = {
|
|
362
|
+
videos: records.length,
|
|
363
|
+
summarized: records.filter((record) => record.status.includes("summarized")).length,
|
|
364
|
+
pending: records.filter((record) => record.status.includes("pending_")).length,
|
|
365
|
+
errors: records.filter((record) => record.status === "error").length,
|
|
366
|
+
summary_files: records.filter((record) => record.status.includes("summarized")).length,
|
|
367
|
+
codex_tokens: records.reduce((total, record) => total + (record.usage?.tokens || 0), 0),
|
|
368
|
+
codex_tokens_unavailable: records.filter((record) => record.status.startsWith("summarized") && record.usage?.tokens === null).length
|
|
369
|
+
};
|
|
370
|
+
const manifest = {
|
|
371
|
+
schema_version: "1.0",
|
|
372
|
+
run: {
|
|
373
|
+
run_date: subtitlesManifest.run?.run_date || options.date,
|
|
374
|
+
input: relative(dirname(manifestPath), inputPath),
|
|
375
|
+
subtitle_policy: "只读取已保存的字幕工件;不下载音频,不执行转写。",
|
|
376
|
+
completed_at: new Date().toISOString()
|
|
377
|
+
},
|
|
378
|
+
summary,
|
|
379
|
+
videos: records
|
|
380
|
+
};
|
|
381
|
+
await writeJsonAtomically(manifestPath, manifest);
|
|
382
|
+
return manifest;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
async function main() {
|
|
386
|
+
const options = parseArgs(process.argv.slice(2));
|
|
387
|
+
if (options.help) return usage();
|
|
388
|
+
const manifest = await runVideoSummaries(options);
|
|
389
|
+
const outputDir = resolve(options.outputDir || join(dirname(resolve(options.input || join("data", "runs", options.date, "subtitles.json"))), "summaries"));
|
|
390
|
+
console.log(`视频摘要索引已保存:${join(dirname(outputDir), "summaries.json")}`);
|
|
391
|
+
console.log(`结果:${manifest.summary.summarized} 已摘要,${manifest.summary.pending} 待处理,${manifest.summary.errors} 失败。`);
|
|
392
|
+
console.log(`模型:${options.model};Codex 报告 token:${manifest.summary.codex_tokens}${manifest.summary.codex_tokens_unavailable ? `(${manifest.summary.codex_tokens_unavailable} 条未报告)` : ""}`);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
396
|
+
main().catch((error) => {
|
|
397
|
+
console.error(`运行失败:${error.message}`);
|
|
398
|
+
process.exitCode = 1;
|
|
399
|
+
});
|
|
400
|
+
}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { pathToFileURL } from "node:url";
|
|
4
|
+
|
|
5
|
+
const CATEGORIES = ["facts", "viewpoints", "demonstrations_or_usage", "limitations"];
|
|
6
|
+
const SUCCESS_STATUSES = new Set(["summarized", "already_summarized", "summarized_with_pending_login_required", "summarized_with_pending_no_subtitles", "summarized_with_pending_subtitle_error", "already_summarized_with_pending_login_required", "already_summarized_with_pending_no_subtitles", "already_summarized_with_pending_subtitle_error"]);
|
|
7
|
+
|
|
8
|
+
function usage() {
|
|
9
|
+
console.log(`
|
|
10
|
+
用法:
|
|
11
|
+
npx ai-weekly synthesize [选项]
|
|
12
|
+
|
|
13
|
+
将同一次运行的项目快照和视频摘要汇总为项目介绍,并生成博主档案。
|
|
14
|
+
|
|
15
|
+
选项:
|
|
16
|
+
--date YYYY-MM-DD 读取和写入 data/runs/<日期>/;默认今天
|
|
17
|
+
--input PATH discovery.json 路径;默认 data/runs/<日期>/discovery.json
|
|
18
|
+
--summaries PATH summaries.json 路径;默认与 discovery.json 同目录
|
|
19
|
+
--creator-labels PATH 人工博主标签 JSON;默认与 discovery.json 同目录的 creator-labels.json
|
|
20
|
+
--output-dir PATH 输出目录;默认与 discovery.json 同目录的 synthesis/
|
|
21
|
+
--help 显示本帮助
|
|
22
|
+
`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function parseArgs(argv) {
|
|
26
|
+
const options = { date: new Date().toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" }), input: null, summaries: null, creatorLabels: null, outputDir: null };
|
|
27
|
+
const valueOptions = new Map([["--date", "date"], ["--input", "input"], ["--summaries", "summaries"], ["--creator-labels", "creatorLabels"], ["--output-dir", "outputDir"]]);
|
|
28
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
29
|
+
const arg = argv[index];
|
|
30
|
+
if (arg === "--help") return { help: true };
|
|
31
|
+
const key = valueOptions.get(arg);
|
|
32
|
+
if (!key || index + 1 >= argv.length) throw new Error(`无效参数:${arg}`);
|
|
33
|
+
options[key] = argv[index + 1];
|
|
34
|
+
index += 1;
|
|
35
|
+
}
|
|
36
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(options.date)) throw new Error("--date 必须为 YYYY-MM-DD");
|
|
37
|
+
return options;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function readJson(path) {
|
|
41
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function readOptionalJson(path) {
|
|
45
|
+
try {
|
|
46
|
+
return await readJson(path);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (error.code === "ENOENT") return { creators: [] };
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function writeJsonAtomically(path, value) {
|
|
54
|
+
const temporaryPath = `${path}.tmp-${process.pid}`;
|
|
55
|
+
await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`);
|
|
56
|
+
await rename(temporaryPath, path);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function uniqueStrings(values) {
|
|
60
|
+
return [...new Set(values.filter((value) => typeof value === "string" && value.trim()).map((value) => value.trim()))];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function projectSnapshot(project) {
|
|
64
|
+
return {
|
|
65
|
+
rank: project.rank,
|
|
66
|
+
repository: project.repository,
|
|
67
|
+
weekly_stars: project.weekly_stars ?? null,
|
|
68
|
+
description: project.description ?? null
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function isSuccessful(record) {
|
|
73
|
+
return SUCCESS_STATUSES.has(record.status) || record.status?.startsWith("summarized");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function hasPendingWork(record) {
|
|
77
|
+
return !isSuccessful(record) || Boolean(record.pending_subtitle_status);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function candidateCount(project) {
|
|
81
|
+
return project.bilibili_search?.recommended_candidates?.length || 0;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function sourceFor(record, artifact) {
|
|
85
|
+
return {
|
|
86
|
+
bvid: record.bvid,
|
|
87
|
+
video_url: record.video_url,
|
|
88
|
+
video_title: record.video_title ?? null,
|
|
89
|
+
...(record.creator ? { creator: record.creator } : {}),
|
|
90
|
+
summary_file: record.summary_file,
|
|
91
|
+
...(record.creator_id ? { creator_id: record.creator_id } : {}),
|
|
92
|
+
...(record.creator_url ? { creator_url: record.creator_url } : {}),
|
|
93
|
+
report: artifact.summary.report
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function pendingFor(record) {
|
|
98
|
+
return {
|
|
99
|
+
bvid: record.bvid,
|
|
100
|
+
status: record.status,
|
|
101
|
+
video_url: record.video_url,
|
|
102
|
+
...(record.error ? { error: record.error } : {})
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function labelMap(labels) {
|
|
107
|
+
return new Map((Array.isArray(labels.creators) ? labels.creators : [])
|
|
108
|
+
.filter((entry) => entry && typeof entry.creator === "string" && entry.creator.trim())
|
|
109
|
+
.map((entry) => [entry.creator.trim(), entry]));
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function readSummary(summaryRoot, record) {
|
|
113
|
+
if (typeof record.summary_file !== "string" || record.summary_file.length === 0) throw new Error("成功摘要记录缺少 summary_file");
|
|
114
|
+
if (isAbsolute(record.summary_file)) throw new Error("摘要工件路径必须是相对路径");
|
|
115
|
+
const path = resolve(summaryRoot, record.summary_file);
|
|
116
|
+
if (path !== summaryRoot && !path.startsWith(`${summaryRoot}${sep}`)) throw new Error("摘要工件路径不能离开运行目录");
|
|
117
|
+
const artifact = await readJson(path);
|
|
118
|
+
const report = artifact.summary?.report;
|
|
119
|
+
if (artifact.schema_version !== "1.0" || artifact.source?.bvid !== record.bvid || !["project_facts", "video_viewpoints"].every((key) => typeof report?.[key] === "string" && report[key].trim())) {
|
|
120
|
+
throw new Error(`摘要工件缺少可用的阅读页摘要:${record.bvid};请使用 npx ai-weekly summarize --force 重新生成`);
|
|
121
|
+
}
|
|
122
|
+
return artifact;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export async function runProjectSynthesis(options) {
|
|
126
|
+
const date = options.date || new Date().toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" });
|
|
127
|
+
const defaultRunDirectory = resolve("data", "runs", date);
|
|
128
|
+
const inputPath = resolve(options.input || join(defaultRunDirectory, "discovery.json"));
|
|
129
|
+
const runDirectory = dirname(inputPath);
|
|
130
|
+
const summariesPath = resolve(options.summaries || join(runDirectory, "summaries.json"));
|
|
131
|
+
const labelsPath = resolve(options.creatorLabels || join(runDirectory, "creator-labels.json"));
|
|
132
|
+
const outputDir = resolve(options.outputDir || join(runDirectory, "synthesis"));
|
|
133
|
+
const discovery = await readJson(inputPath);
|
|
134
|
+
const summaries = await readJson(summariesPath);
|
|
135
|
+
if (!Array.isArray(discovery.projects)) throw new Error("discovery.json 缺少 projects");
|
|
136
|
+
if (!Array.isArray(summaries.videos)) throw new Error("summaries.json 缺少 videos");
|
|
137
|
+
const labels = labelMap(await readOptionalJson(labelsPath));
|
|
138
|
+
const recordsByRepository = new Map();
|
|
139
|
+
for (const record of summaries.videos) {
|
|
140
|
+
if (!recordsByRepository.has(record.repository)) recordsByRepository.set(record.repository, []);
|
|
141
|
+
recordsByRepository.get(record.repository).push(record);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const creators = new Map();
|
|
145
|
+
const projects = [];
|
|
146
|
+
for (const project of discovery.projects) {
|
|
147
|
+
const records = recordsByRepository.get(project.repository) || [];
|
|
148
|
+
const successful = records.filter(isSuccessful);
|
|
149
|
+
const pending = records.filter(hasPendingWork);
|
|
150
|
+
const introduction = Object.fromEntries(CATEGORIES.map((category) => [category, []]));
|
|
151
|
+
const sources = { videos: [] };
|
|
152
|
+
for (const record of successful) {
|
|
153
|
+
const artifact = await readSummary(dirname(summariesPath), record);
|
|
154
|
+
for (const category of CATEGORIES) introduction[category].push(...(artifact.summary[category] || []));
|
|
155
|
+
sources.videos.push(sourceFor(record, artifact));
|
|
156
|
+
if (!record.creator) continue;
|
|
157
|
+
const creatorKey = record.creator_id || record.creator;
|
|
158
|
+
const profile = creators.get(creatorKey) || {
|
|
159
|
+
creator: record.creator,
|
|
160
|
+
...(record.creator_id ? { creator_id: record.creator_id } : {}),
|
|
161
|
+
...(record.creator_url ? { profile_url: record.creator_url } : {}),
|
|
162
|
+
processed_video_count: 0,
|
|
163
|
+
project_count: 0,
|
|
164
|
+
projects: [],
|
|
165
|
+
videos: []
|
|
166
|
+
};
|
|
167
|
+
profile.processed_video_count += 1;
|
|
168
|
+
if (!profile.projects.includes(project.repository)) profile.projects.push(project.repository);
|
|
169
|
+
profile.videos.push({ repository: project.repository, bvid: record.bvid, video_url: record.video_url, status: record.status });
|
|
170
|
+
if (!profile.profile_url && record.creator_url) profile.profile_url = record.creator_url;
|
|
171
|
+
creators.set(creatorKey, profile);
|
|
172
|
+
}
|
|
173
|
+
for (const category of CATEGORIES) introduction[category] = uniqueStrings(introduction[category]);
|
|
174
|
+
const processing = { candidate_count: candidateCount(project), summarized_count: successful.length, pending_count: pending.length, pending: pending.map(pendingFor) };
|
|
175
|
+
projects.push({
|
|
176
|
+
repository: project.repository,
|
|
177
|
+
status: pending.length === 0 && successful.length > 0 && candidateCount(project) > 0 ? "complete" : "incomplete",
|
|
178
|
+
project_snapshot: projectSnapshot(project),
|
|
179
|
+
introduction,
|
|
180
|
+
sources,
|
|
181
|
+
processing
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const creatorProfiles = [...creators.values()].map((profile) => {
|
|
186
|
+
const label = labels.get(profile.creator);
|
|
187
|
+
return {
|
|
188
|
+
creator: profile.creator,
|
|
189
|
+
...(typeof label?.quality_label === "string" && label.quality_label.trim() ? { quality_label: label.quality_label.trim(), label_source: "manual" } : {}),
|
|
190
|
+
...profile,
|
|
191
|
+
project_count: profile.projects.length
|
|
192
|
+
};
|
|
193
|
+
});
|
|
194
|
+
const result = {
|
|
195
|
+
schema_version: "1.0",
|
|
196
|
+
run: { run_date: discovery.run?.run_date || date, discovery: relative(outputDir, inputPath), summaries: relative(outputDir, summariesPath), completed_at: new Date().toISOString() },
|
|
197
|
+
summary: {
|
|
198
|
+
projects: projects.length,
|
|
199
|
+
complete: projects.filter((project) => project.status === "complete").length,
|
|
200
|
+
incomplete: projects.filter((project) => project.status === "incomplete").length,
|
|
201
|
+
summarized_videos: projects.reduce((total, project) => total + project.processing.summarized_count, 0),
|
|
202
|
+
pending_videos: projects.reduce((total, project) => total + project.processing.pending_count, 0),
|
|
203
|
+
creators: creatorProfiles.length
|
|
204
|
+
},
|
|
205
|
+
projects,
|
|
206
|
+
creators: creatorProfiles
|
|
207
|
+
};
|
|
208
|
+
await mkdir(outputDir, { recursive: true });
|
|
209
|
+
const reportProjects = projects.map((project) => {
|
|
210
|
+
const videos = [];
|
|
211
|
+
for (const source of project.sources.videos) {
|
|
212
|
+
videos.push({ bvid: source.bvid, video_url: source.video_url, video_title: source.video_title, creator: source.creator || "", status: "included", summary: { project_facts: source.report.project_facts, video_viewpoints: source.report.video_viewpoints } });
|
|
213
|
+
}
|
|
214
|
+
return { repository: project.repository, status: project.status, project_snapshot: project.project_snapshot, videos };
|
|
215
|
+
});
|
|
216
|
+
const reportContent = {
|
|
217
|
+
schema_version: "1.0",
|
|
218
|
+
run: { run_date: result.run.run_date, input: "project-synthesis.json", completed_at: result.run.completed_at },
|
|
219
|
+
summary: {
|
|
220
|
+
projects: reportProjects.length,
|
|
221
|
+
videos: reportProjects.reduce((total, project) => total + project.videos.length, 0),
|
|
222
|
+
errors: 0,
|
|
223
|
+
codex_tokens: 0,
|
|
224
|
+
codex_tokens_unavailable: 0
|
|
225
|
+
},
|
|
226
|
+
projects: reportProjects
|
|
227
|
+
};
|
|
228
|
+
await writeJsonAtomically(join(outputDir, "project-synthesis.json"), result);
|
|
229
|
+
await writeJsonAtomically(join(outputDir, "creator-profiles.json"), { schema_version: "1.0", creators: creatorProfiles });
|
|
230
|
+
await writeJsonAtomically(join(outputDir, "report-content.json"), reportContent);
|
|
231
|
+
return result;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function main() {
|
|
235
|
+
const options = parseArgs(process.argv.slice(2));
|
|
236
|
+
if (options.help) return usage();
|
|
237
|
+
const result = await runProjectSynthesis(options);
|
|
238
|
+
const inputPath = resolve(options.input || join("data", "runs", options.date, "discovery.json"));
|
|
239
|
+
const outputDir = resolve(options.outputDir || join(dirname(inputPath), "synthesis"));
|
|
240
|
+
console.log(`项目汇总已保存:${join(outputDir, "project-synthesis.json")}`);
|
|
241
|
+
console.log(`结果:${result.summary.projects} 个项目,${result.summary.summarized_videos} 条已摘要,${result.summary.pending_videos} 条待处理,${result.summary.creators} 位博主。`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
245
|
+
main().catch((error) => { console.error(`运行失败:${error.message}`); process.exitCode = 1; });
|
|
246
|
+
}
|