@s-hirano-ist/s-scripts 1.26.8 ā 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/chunk-5OXUZW2Q.js +590 -0
- package/dist/chunk-7PVU2Z32.js +71 -0
- package/dist/chunk-AJMTRCPJ.js +15 -0
- package/dist/chunk-AZF2R2BF.js +85 -0
- package/dist/chunk-F4J6JBR4.js +115 -0
- package/dist/chunk-HDYQV2Y2.js +58 -0
- package/dist/chunk-HIK7SX34.js +85 -0
- package/dist/chunk-IWNZQXDG.js +7660 -0
- package/dist/chunk-J4VLH4VA.js +267 -0
- package/dist/chunk-JZOR5GA2.js +42 -0
- package/dist/chunk-KKZP7ZF3.js +77 -0
- package/dist/chunk-N6TZF755.js +102 -0
- package/dist/chunk-S5WJ5VXW.js +85 -0
- package/dist/chunk-UXGLXHQ6.js +40 -0
- package/dist/cleanup-minio-images.js +115 -92
- package/dist/enrich-books.js +87 -78
- package/dist/fetch-articles.js +101 -90
- package/dist/fetch-books.js +147 -128
- package/dist/fetch-images.js +88 -72
- package/dist/fetch-notes.js +66 -54
- package/dist/find-duplicate-json-articles.js +100 -97
- package/dist/fix-book-frontmatter.js +111 -99
- package/dist/fix-image-formats.js +81 -65
- package/dist/ingest-articles.js +30 -249
- package/dist/ingest-books.js +61 -318
- package/dist/ingest-images.js +33 -184
- package/dist/ingest-notes.js +41 -171
- package/dist/rag/delete-collection.js +32 -22
- package/dist/rag/ingest.js +371 -92
- package/dist/rag/search.js +80 -76
- package/dist/reset-articles.js +57 -60
- package/dist/reset-books.js +57 -59
- package/dist/reset-images.js +57 -59
- package/dist/reset-notes.js +57 -59
- package/dist/revert-articles.js +53 -55
- package/dist/revert-books.js +53 -53
- package/dist/revert-images.js +53 -53
- package/dist/revert-notes.js +53 -53
- package/dist/s-content-format.js +103 -13
- package/dist/update-json-articles.js +101 -97
- package/dist/update-raw-articles.js +130 -130
- package/dist/validate-json-articles.js +129 -130
- package/package.json +13 -50
- package/dist/cleanup-minio-images.d.ts +0 -3
- package/dist/cleanup-minio-images.d.ts.map +0 -1
- package/dist/cleanup-minio-images.js.map +0 -1
- package/dist/enrich-books.d.ts +0 -3
- package/dist/enrich-books.d.ts.map +0 -1
- package/dist/enrich-books.js.map +0 -1
- package/dist/fetch-articles.d.ts +0 -3
- package/dist/fetch-articles.d.ts.map +0 -1
- package/dist/fetch-articles.js.map +0 -1
- package/dist/fetch-books.d.ts +0 -3
- package/dist/fetch-books.d.ts.map +0 -1
- package/dist/fetch-books.js.map +0 -1
- package/dist/fetch-images.d.ts +0 -3
- package/dist/fetch-images.d.ts.map +0 -1
- package/dist/fetch-images.js.map +0 -1
- package/dist/fetch-notes.d.ts +0 -3
- package/dist/fetch-notes.d.ts.map +0 -1
- package/dist/fetch-notes.js.map +0 -1
- package/dist/find-duplicate-json-articles.d.ts +0 -3
- package/dist/find-duplicate-json-articles.d.ts.map +0 -1
- package/dist/find-duplicate-json-articles.js.map +0 -1
- package/dist/fix-book-frontmatter.d.ts +0 -3
- package/dist/fix-book-frontmatter.d.ts.map +0 -1
- package/dist/fix-book-frontmatter.js.map +0 -1
- package/dist/fix-image-formats.d.ts +0 -3
- package/dist/fix-image-formats.d.ts.map +0 -1
- package/dist/fix-image-formats.js.map +0 -1
- package/dist/infrastructures/articles-command-repository.d.ts +0 -57
- package/dist/infrastructures/articles-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/articles-command-repository.js +0 -68
- package/dist/infrastructures/articles-command-repository.js.map +0 -1
- package/dist/infrastructures/books-command-repository.d.ts +0 -34
- package/dist/infrastructures/books-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/books-command-repository.js +0 -53
- package/dist/infrastructures/books-command-repository.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.d.ts +0 -9
- package/dist/infrastructures/cache-invalidation-client.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.js +0 -21
- package/dist/infrastructures/cache-invalidation-client.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.d.ts +0 -2
- package/dist/infrastructures/cache-invalidation-client.test.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.js +0 -25
- package/dist/infrastructures/cache-invalidation-client.test.js.map +0 -1
- package/dist/infrastructures/images-command-repository.d.ts +0 -34
- package/dist/infrastructures/images-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/images-command-repository.js +0 -53
- package/dist/infrastructures/images-command-repository.js.map +0 -1
- package/dist/infrastructures/notes-command-repository.d.ts +0 -34
- package/dist/infrastructures/notes-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/notes-command-repository.js +0 -53
- package/dist/infrastructures/notes-command-repository.js.map +0 -1
- package/dist/ingest-articles.d.ts +0 -3
- package/dist/ingest-articles.d.ts.map +0 -1
- package/dist/ingest-articles.js.map +0 -1
- package/dist/ingest-books.d.ts +0 -3
- package/dist/ingest-books.d.ts.map +0 -1
- package/dist/ingest-books.js.map +0 -1
- package/dist/ingest-images.d.ts +0 -3
- package/dist/ingest-images.d.ts.map +0 -1
- package/dist/ingest-images.js.map +0 -1
- package/dist/ingest-notes.d.ts +0 -3
- package/dist/ingest-notes.d.ts.map +0 -1
- package/dist/ingest-notes.js.map +0 -1
- package/dist/lib/glob-paths.d.ts +0 -6
- package/dist/lib/glob-paths.d.ts.map +0 -1
- package/dist/lib/glob-paths.js +0 -9
- package/dist/lib/glob-paths.js.map +0 -1
- package/dist/lib/glob-paths.test.d.ts +0 -2
- package/dist/lib/glob-paths.test.d.ts.map +0 -1
- package/dist/lib/glob-paths.test.js +0 -30
- package/dist/lib/glob-paths.test.js.map +0 -1
- package/dist/lib/html-charset.d.ts +0 -4
- package/dist/lib/html-charset.d.ts.map +0 -1
- package/dist/lib/html-charset.js +0 -33
- package/dist/lib/html-charset.js.map +0 -1
- package/dist/lib/html-charset.test.d.ts +0 -2
- package/dist/lib/html-charset.test.d.ts.map +0 -1
- package/dist/lib/html-charset.test.js +0 -27
- package/dist/lib/html-charset.test.js.map +0 -1
- package/dist/rag/delete-collection.d.ts +0 -3
- package/dist/rag/delete-collection.d.ts.map +0 -1
- package/dist/rag/delete-collection.js.map +0 -1
- package/dist/rag/ingest-config.d.ts +0 -8
- package/dist/rag/ingest-config.d.ts.map +0 -1
- package/dist/rag/ingest-config.js +0 -8
- package/dist/rag/ingest-config.js.map +0 -1
- package/dist/rag/ingest.d.ts +0 -3
- package/dist/rag/ingest.d.ts.map +0 -1
- package/dist/rag/ingest.js.map +0 -1
- package/dist/rag/search.d.ts +0 -3
- package/dist/rag/search.d.ts.map +0 -1
- package/dist/rag/search.js.map +0 -1
- package/dist/reset-articles.d.ts +0 -3
- package/dist/reset-articles.d.ts.map +0 -1
- package/dist/reset-articles.js.map +0 -1
- package/dist/reset-books.d.ts +0 -3
- package/dist/reset-books.d.ts.map +0 -1
- package/dist/reset-books.js.map +0 -1
- package/dist/reset-images.d.ts +0 -3
- package/dist/reset-images.d.ts.map +0 -1
- package/dist/reset-images.js.map +0 -1
- package/dist/reset-notes.d.ts +0 -3
- package/dist/reset-notes.d.ts.map +0 -1
- package/dist/reset-notes.js.map +0 -1
- package/dist/revert-articles.d.ts +0 -3
- package/dist/revert-articles.d.ts.map +0 -1
- package/dist/revert-articles.js.map +0 -1
- package/dist/revert-books.d.ts +0 -3
- package/dist/revert-books.d.ts.map +0 -1
- package/dist/revert-books.js.map +0 -1
- package/dist/revert-images.d.ts +0 -3
- package/dist/revert-images.d.ts.map +0 -1
- package/dist/revert-images.js.map +0 -1
- package/dist/revert-notes.d.ts +0 -3
- package/dist/revert-notes.d.ts.map +0 -1
- package/dist/revert-notes.js.map +0 -1
- package/dist/s-content-format-runner.d.ts +0 -10
- package/dist/s-content-format-runner.d.ts.map +0 -1
- package/dist/s-content-format-runner.js +0 -76
- package/dist/s-content-format-runner.js.map +0 -1
- package/dist/s-content-format.d.ts +0 -3
- package/dist/s-content-format.d.ts.map +0 -1
- package/dist/s-content-format.js.map +0 -1
- package/dist/s-content-format.test.d.ts +0 -2
- package/dist/s-content-format.test.d.ts.map +0 -1
- package/dist/s-content-format.test.js +0 -99
- package/dist/s-content-format.test.js.map +0 -1
- package/dist/update-json-articles.d.ts +0 -3
- package/dist/update-json-articles.d.ts.map +0 -1
- package/dist/update-json-articles.js.map +0 -1
- package/dist/update-raw-articles.d.ts +0 -3
- package/dist/update-raw-articles.d.ts.map +0 -1
- package/dist/update-raw-articles.js.map +0 -1
- package/dist/validate-json-articles.d.ts +0 -3
- package/dist/validate-json-articles.d.ts.map +0 -1
- package/dist/validate-json-articles.js.map +0 -1
package/dist/ingest-articles.js
CHANGED
|
@@ -1,252 +1,33 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
runContentImport
|
|
5
|
+
} from "./chunk-5OXUZW2Q.js";
|
|
6
|
+
import "./chunk-KKZP7ZF3.js";
|
|
7
|
+
import "./chunk-F4J6JBR4.js";
|
|
8
|
+
import "./chunk-IWNZQXDG.js";
|
|
9
|
+
import "./chunk-7PVU2Z32.js";
|
|
10
|
+
import {
|
|
11
|
+
globPaths
|
|
12
|
+
} from "./chunk-AJMTRCPJ.js";
|
|
13
|
+
import "./chunk-JZOR5GA2.js";
|
|
14
|
+
|
|
15
|
+
// src/ingest-articles.ts
|
|
5
16
|
import { readFile } from "node:fs/promises";
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
async function* source(root) {
|
|
18
|
+
for (const path of await globPaths(`${root}/json/article/*.json`)) {
|
|
19
|
+
const json = JSON.parse(await readFile(path, "utf-8"));
|
|
20
|
+
for (const item of json.body)
|
|
21
|
+
yield {
|
|
22
|
+
values: {
|
|
23
|
+
...item,
|
|
24
|
+
categoryName: json.heading,
|
|
25
|
+
quote: item.quote ?? null,
|
|
26
|
+
ogTitle: item.ogTitle ?? null,
|
|
27
|
+
ogDescription: item.ogDescription ?? null,
|
|
28
|
+
ogImageUrl: item.ogImageUrl ?? null
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
15
32
|
}
|
|
16
|
-
|
|
17
|
-
if (outcome === "inserted")
|
|
18
|
-
return "insertedCount";
|
|
19
|
-
if (outcome === "updated")
|
|
20
|
-
return "updatedCount";
|
|
21
|
-
return "skippedCount";
|
|
22
|
-
}
|
|
23
|
-
async function main() {
|
|
24
|
-
const dryRun = process.argv.includes("--dry-run");
|
|
25
|
-
const env = {
|
|
26
|
-
DATABASE_URL: process.env.DATABASE_URL,
|
|
27
|
-
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
28
|
-
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
29
|
-
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
30
|
-
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
31
|
-
};
|
|
32
|
-
if (Object.values(env).some((v) => !v)) {
|
|
33
|
-
throw new Error("Required environment variables are not set.");
|
|
34
|
-
}
|
|
35
|
-
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
36
|
-
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
37
|
-
const notificationService = createPushoverService({
|
|
38
|
-
url: env.PUSHOVER_URL ?? "",
|
|
39
|
-
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
40
|
-
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
41
|
-
});
|
|
42
|
-
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
43
|
-
const exported = makeExportedStatus();
|
|
44
|
-
const fileUrls = new Set();
|
|
45
|
-
async function processExisting(existing, item) {
|
|
46
|
-
const fileQuote = item.quote ?? null;
|
|
47
|
-
const fileOgImageUrl = item.ogImageUrl ?? null;
|
|
48
|
-
const fileOgTitle = item.ogTitle ?? null;
|
|
49
|
-
const fileOgDescription = item.ogDescription ?? null;
|
|
50
|
-
if (isUnchanged(existing, item, fileQuote, fileOgImageUrl, fileOgTitle, fileOgDescription)) {
|
|
51
|
-
// console.log(
|
|
52
|
-
// `āļø ć¹ćććļ¼å¤ę“ćŖćļ¼: ${item.title} (${item.url})`,
|
|
53
|
-
// );
|
|
54
|
-
return "skipped";
|
|
55
|
-
}
|
|
56
|
-
if (dryRun) {
|
|
57
|
-
console.log(`š [dry-run] ę“ę°äŗå®: ${item.title} (${item.url})`);
|
|
58
|
-
}
|
|
59
|
-
else {
|
|
60
|
-
await prisma.article.update({
|
|
61
|
-
where: { id: existing.id },
|
|
62
|
-
data: {
|
|
63
|
-
title: item.title,
|
|
64
|
-
quote: fileQuote,
|
|
65
|
-
ogImageUrl: fileOgImageUrl,
|
|
66
|
-
ogTitle: fileOgTitle,
|
|
67
|
-
ogDescription: fileOgDescription,
|
|
68
|
-
},
|
|
69
|
-
});
|
|
70
|
-
console.log(`š ę“ę°: ${item.title}`);
|
|
71
|
-
}
|
|
72
|
-
return "updated";
|
|
73
|
-
}
|
|
74
|
-
async function processNew(item, categoryId, existingArticlesMap) {
|
|
75
|
-
if (dryRun) {
|
|
76
|
-
console.log(`š [dry-run] ęæå
„äŗå®: ${item.title} (${item.url})`);
|
|
77
|
-
existingArticlesMap.set(item.url, {
|
|
78
|
-
id: "",
|
|
79
|
-
url: item.url,
|
|
80
|
-
title: item.title,
|
|
81
|
-
quote: item.quote ?? null,
|
|
82
|
-
ogImageUrl: item.ogImageUrl ?? null,
|
|
83
|
-
ogTitle: item.ogTitle ?? null,
|
|
84
|
-
ogDescription: item.ogDescription ?? null,
|
|
85
|
-
});
|
|
86
|
-
return "inserted";
|
|
87
|
-
}
|
|
88
|
-
await prisma.article.create({
|
|
89
|
-
data: {
|
|
90
|
-
id: String(makeId()),
|
|
91
|
-
title: item.title,
|
|
92
|
-
url: item.url,
|
|
93
|
-
quote: item.quote ?? null,
|
|
94
|
-
ogImageUrl: item.ogImageUrl ?? null,
|
|
95
|
-
ogTitle: item.ogTitle ?? null,
|
|
96
|
-
ogDescription: item.ogDescription ?? null,
|
|
97
|
-
categoryId,
|
|
98
|
-
status: exported.status,
|
|
99
|
-
exportedAt: exported.exportedAt,
|
|
100
|
-
userId,
|
|
101
|
-
createdAt: new Date(),
|
|
102
|
-
},
|
|
103
|
-
});
|
|
104
|
-
existingArticlesMap.set(item.url, {
|
|
105
|
-
id: "",
|
|
106
|
-
url: item.url,
|
|
107
|
-
title: item.title,
|
|
108
|
-
quote: item.quote ?? null,
|
|
109
|
-
ogImageUrl: item.ogImageUrl ?? null,
|
|
110
|
-
ogTitle: item.ogTitle ?? null,
|
|
111
|
-
ogDescription: item.ogDescription ?? null,
|
|
112
|
-
});
|
|
113
|
-
console.log(`ā
ęæå
„: ${item.title}`);
|
|
114
|
-
return "inserted";
|
|
115
|
-
}
|
|
116
|
-
async function processItem(item, categoryId, existingArticlesMap) {
|
|
117
|
-
fileUrls.add(item.url);
|
|
118
|
-
const existing = existingArticlesMap.get(item.url);
|
|
119
|
-
if (existing) {
|
|
120
|
-
return processExisting(existing, item);
|
|
121
|
-
}
|
|
122
|
-
return processNew(item, categoryId, existingArticlesMap);
|
|
123
|
-
}
|
|
124
|
-
async function resolveCategoryId(categoryName, categoryMap) {
|
|
125
|
-
const existingId = categoryMap.get(categoryName);
|
|
126
|
-
if (existingId) {
|
|
127
|
-
return { categoryId: existingId, created: false };
|
|
128
|
-
}
|
|
129
|
-
let categoryId;
|
|
130
|
-
if (dryRun) {
|
|
131
|
-
console.log(`š [dry-run] ć«ćć“ćŖä½ęäŗå®: ${categoryName}`);
|
|
132
|
-
categoryId = `dry-run-${categoryName}`;
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
const category = await prisma.category.create({
|
|
136
|
-
data: {
|
|
137
|
-
id: String(makeId()),
|
|
138
|
-
name: categoryName,
|
|
139
|
-
userId,
|
|
140
|
-
createdAt: new Date(),
|
|
141
|
-
},
|
|
142
|
-
});
|
|
143
|
-
categoryId = category.id;
|
|
144
|
-
console.log(`š ć«ćć“ćŖä½ę: ${categoryName}`);
|
|
145
|
-
}
|
|
146
|
-
categoryMap.set(categoryName, categoryId);
|
|
147
|
-
return { categoryId, created: true };
|
|
148
|
-
}
|
|
149
|
-
async function ingestArticles() {
|
|
150
|
-
const files = await globPaths(`${contentsPath}/json/article/*.json`);
|
|
151
|
-
console.log(`š ${files.length} ä»¶ć®JSONćć”ć¤ć«ćę¤åŗćć¾ććć`);
|
|
152
|
-
const existingArticles = await prisma.article.findMany({
|
|
153
|
-
where: { userId },
|
|
154
|
-
select: {
|
|
155
|
-
id: true,
|
|
156
|
-
url: true,
|
|
157
|
-
title: true,
|
|
158
|
-
quote: true,
|
|
159
|
-
ogImageUrl: true,
|
|
160
|
-
ogTitle: true,
|
|
161
|
-
ogDescription: true,
|
|
162
|
-
},
|
|
163
|
-
});
|
|
164
|
-
const existingArticlesMap = new Map(existingArticles.map((a) => [a.url, a]));
|
|
165
|
-
console.log(`š DB ć« ${existingArticlesMap.size} ä»¶ć®ę¢åčØäŗćććć¾ćć`);
|
|
166
|
-
const existingCategories = await prisma.category.findMany({
|
|
167
|
-
where: { userId },
|
|
168
|
-
});
|
|
169
|
-
const categoryMap = new Map(existingCategories.map((c) => [c.name, c.id]));
|
|
170
|
-
const counts = { insertedCount: 0, updatedCount: 0, skippedCount: 0 };
|
|
171
|
-
let errorCount = 0;
|
|
172
|
-
let categoryCreatedCount = 0;
|
|
173
|
-
for (const filePath of files) {
|
|
174
|
-
try {
|
|
175
|
-
const content = await readFile(filePath, "utf-8");
|
|
176
|
-
const json = JSON.parse(content);
|
|
177
|
-
const categoryName = json.heading;
|
|
178
|
-
const { categoryId, created } = await resolveCategoryId(categoryName, categoryMap);
|
|
179
|
-
if (created) {
|
|
180
|
-
categoryCreatedCount++;
|
|
181
|
-
}
|
|
182
|
-
for (const item of json.body) {
|
|
183
|
-
try {
|
|
184
|
-
const outcome = await processItem(item, categoryId, existingArticlesMap);
|
|
185
|
-
counts[outcomeKey(outcome)]++;
|
|
186
|
-
}
|
|
187
|
-
catch (itemError) {
|
|
188
|
-
console.error(`ā ćØć©ć¼ļ¼${basename(filePath)} > ${item.title}ļ¼:`, `url(${item.url.length}ęå)=${item.url}`, `title(${item.title.length}ęå)`, `quote(${(item.quote ?? "").length}ęå)`, `ogImageUrl(${(item.ogImageUrl ?? "").length}ęå)`, `ogTitle(${(item.ogTitle ?? "").length}ęå)`, `ogDescription(${(item.ogDescription ?? "").length}ęå)`, itemError);
|
|
189
|
-
errorCount++;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
catch (error) {
|
|
194
|
-
console.error(`ā ćØć©ć¼ļ¼${basename(filePath)}ļ¼:`, error);
|
|
195
|
-
errorCount++;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
return {
|
|
199
|
-
insertedCount: counts.insertedCount,
|
|
200
|
-
updatedCount: counts.updatedCount,
|
|
201
|
-
skippedCount: counts.skippedCount,
|
|
202
|
-
errorCount,
|
|
203
|
-
categoryCreatedCount,
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
async function purgeArticles() {
|
|
207
|
-
const exportedArticles = await prisma.article.findMany({
|
|
208
|
-
where: { userId, status: exported.status },
|
|
209
|
-
select: { id: true, url: true },
|
|
210
|
-
});
|
|
211
|
-
const toDelete = exportedArticles.filter((a) => !fileUrls.has(a.url));
|
|
212
|
-
if (toDelete.length === 0) {
|
|
213
|
-
console.log("šļø åé¤åÆ¾č±”ćŖć");
|
|
214
|
-
return 0;
|
|
215
|
-
}
|
|
216
|
-
let deletedCount = 0;
|
|
217
|
-
for (const article of toDelete) {
|
|
218
|
-
if (dryRun) {
|
|
219
|
-
console.log(`šļø [dry-run] åé¤äŗå®: ${article.url}`);
|
|
220
|
-
}
|
|
221
|
-
else {
|
|
222
|
-
await prisma.article.delete({ where: { id: article.id } });
|
|
223
|
-
console.log(`šļø åé¤: ${article.url}`);
|
|
224
|
-
}
|
|
225
|
-
deletedCount++;
|
|
226
|
-
}
|
|
227
|
-
return deletedCount;
|
|
228
|
-
}
|
|
229
|
-
try {
|
|
230
|
-
const { insertedCount, updatedCount, skippedCount, errorCount, categoryCreatedCount, } = await ingestArticles();
|
|
231
|
-
const deletedCount = await purgeArticles();
|
|
232
|
-
console.log(`\nš ēµę: ęæå
„ ${insertedCount} ä»¶, ę“ę° ${updatedCount} ä»¶, ć¹ććć ${skippedCount} ä»¶, åé¤ ${deletedCount} ä»¶, ćØć©ć¼ ${errorCount} ä»¶, ć«ćć“ćŖę°č¦ ${categoryCreatedCount} ä»¶${dryRun ? " (dry-run)" : ""}`);
|
|
233
|
-
await notificationService.notifyInfo(`${SCRIPT_NAME} completed`, {
|
|
234
|
-
caller: SCRIPT_NAME,
|
|
235
|
-
});
|
|
236
|
-
}
|
|
237
|
-
catch (error) {
|
|
238
|
-
console.error("ā ćØć©ć¼ćēŗēćć¾ćć:", error);
|
|
239
|
-
await notificationService.notifyError(`${SCRIPT_NAME} failed: ${String(error)}`, {
|
|
240
|
-
caller: SCRIPT_NAME,
|
|
241
|
-
});
|
|
242
|
-
process.exit(1);
|
|
243
|
-
}
|
|
244
|
-
finally {
|
|
245
|
-
await prisma.$disconnect();
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
main().catch((error) => {
|
|
249
|
-
console.error(error);
|
|
250
|
-
process.exit(1);
|
|
251
|
-
});
|
|
252
|
-
//# sourceMappingURL=ingest-articles.js.map
|
|
33
|
+
await runContentImport("articles", source);
|
package/dist/ingest-books.js
CHANGED
|
@@ -1,325 +1,68 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
import
|
|
8
|
-
import
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
makeISBN,
|
|
5
|
+
runContentImport
|
|
6
|
+
} from "./chunk-5OXUZW2Q.js";
|
|
7
|
+
import "./chunk-KKZP7ZF3.js";
|
|
8
|
+
import "./chunk-F4J6JBR4.js";
|
|
9
|
+
import "./chunk-IWNZQXDG.js";
|
|
10
|
+
import "./chunk-7PVU2Z32.js";
|
|
11
|
+
import {
|
|
12
|
+
parseMarkdownFrontmatter
|
|
13
|
+
} from "./chunk-HDYQV2Y2.js";
|
|
14
|
+
import {
|
|
15
|
+
globPaths
|
|
16
|
+
} from "./chunk-AJMTRCPJ.js";
|
|
17
|
+
import "./chunk-JZOR5GA2.js";
|
|
18
|
+
|
|
19
|
+
// src/ingest-books.ts
|
|
9
20
|
import { readFile } from "node:fs/promises";
|
|
10
21
|
import { basename, extname } from "node:path";
|
|
11
|
-
import { globPaths } from "./lib/glob-paths.js";
|
|
12
|
-
const SCRIPT_NAME = "ingest-books";
|
|
13
|
-
const ORIGINAL_BOOK_IMAGE_PATH = "books/original";
|
|
14
|
-
const THUMBNAIL_BOOK_IMAGE_PATH = "books/thumbnail";
|
|
15
|
-
const THUMBNAIL_SIZE = 192;
|
|
16
|
-
function getContentType(filePath) {
|
|
17
|
-
const ext = extname(filePath).toLowerCase();
|
|
18
|
-
switch (ext) {
|
|
19
|
-
case ".jpg":
|
|
20
|
-
case ".jpeg":
|
|
21
|
-
return "image/jpeg";
|
|
22
|
-
case ".png":
|
|
23
|
-
return "image/png";
|
|
24
|
-
case ".webp":
|
|
25
|
-
return "image/webp";
|
|
26
|
-
default:
|
|
27
|
-
return null;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
function arrayEquals(a, b) {
|
|
31
|
-
if (a.length !== b.length)
|
|
32
|
-
return false;
|
|
33
|
-
for (const [i, element] of a.entries()) {
|
|
34
|
-
if (element !== b[i])
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
return true;
|
|
38
|
-
}
|
|
39
22
|
function parseBookFile(content) {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
function bookMatchesExisting(existing, parsed, expectedImagePath) {
|
|
66
|
-
return (existing.title === parsed.title &&
|
|
67
|
-
existing.markdown === parsed.markdown &&
|
|
68
|
-
existing.imagePath === expectedImagePath &&
|
|
69
|
-
existing.rating === parsed.rating &&
|
|
70
|
-
arrayEquals(existing.tags, parsed.tags) &&
|
|
71
|
-
existing.googleSubTitle === parsed.googleSubTitle &&
|
|
72
|
-
arrayEquals(existing.googleAuthors, parsed.googleAuthors) &&
|
|
73
|
-
existing.googleDescription === parsed.googleDescription &&
|
|
74
|
-
existing.googleImgSrc === parsed.googleImgSrc &&
|
|
75
|
-
existing.googleHref === parsed.googleHref);
|
|
23
|
+
const parsed = parseMarkdownFrontmatter(content);
|
|
24
|
+
const data = parsed.data;
|
|
25
|
+
let title = data.title?.trim() ?? "";
|
|
26
|
+
if (!title) {
|
|
27
|
+
const h1Match = /^# (.+)$/mu.exec(parsed.content);
|
|
28
|
+
title = h1Match ? h1Match[1].trim() : "";
|
|
29
|
+
}
|
|
30
|
+
const body = parsed.content.trim();
|
|
31
|
+
if (data.rating === void 0 || data.rating === null) {
|
|
32
|
+
throw new Error(
|
|
33
|
+
"rating \u306F\u5FC5\u9808\u3067\u3059 (frontmatter \u306B rating \u30D5\u30A3\u30FC\u30EB\u30C9\u304C\u3042\u308A\u307E\u305B\u3093)"
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
title,
|
|
38
|
+
markdown: body || null,
|
|
39
|
+
rating: data.rating,
|
|
40
|
+
tags: data.tags ?? [],
|
|
41
|
+
googleSubTitle: data.googleSubtitle ?? null,
|
|
42
|
+
googleAuthors: data.googleAuthors ?? [],
|
|
43
|
+
googleDescription: data.googleDescription ?? null,
|
|
44
|
+
googleImgSrc: data.googleImgSrc ?? null,
|
|
45
|
+
googleHref: data.googleHref ?? null
|
|
46
|
+
};
|
|
76
47
|
}
|
|
77
|
-
async function
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
48
|
+
async function* source(root) {
|
|
49
|
+
const covers = /* @__PURE__ */ new Map();
|
|
50
|
+
for (const path of await globPaths(`${root}/image/book/*`)) {
|
|
51
|
+
if (!/\.(?:jpg|jpeg|png|webp)$/iu.test(path)) continue;
|
|
52
|
+
covers.set(makeISBN(basename(path, extname(path))), path);
|
|
53
|
+
}
|
|
54
|
+
for (const path of await globPaths(`${root}/markdown/book/*.md`)) {
|
|
55
|
+
const isbn = makeISBN(basename(path, ".md"));
|
|
56
|
+
const cover = covers.get(isbn);
|
|
57
|
+
yield {
|
|
58
|
+
values: {
|
|
59
|
+
...parseBookFile(await readFile(path, "utf-8")),
|
|
60
|
+
isbn,
|
|
61
|
+
imagePath: cover ? basename(cover) : null
|
|
62
|
+
},
|
|
63
|
+
preserveImagePath: !cover,
|
|
64
|
+
...cover ? { media: { path: basename(cover), bytes: await readFile(cover) } } : {}
|
|
86
65
|
};
|
|
87
|
-
|
|
88
|
-
throw new Error("Required environment variables are not set.");
|
|
89
|
-
}
|
|
90
|
-
if (process.env.MINIO_USE_SSL === "true" &&
|
|
91
|
-
(!process.env.CF_ACCESS_CLIENT_ID || !process.env.CF_ACCESS_CLIENT_SECRET)) {
|
|
92
|
-
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
|
|
93
|
-
}
|
|
94
|
-
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
95
|
-
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
96
|
-
const notificationService = createPushoverService({
|
|
97
|
-
url: env.PUSHOVER_URL ?? "",
|
|
98
|
-
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
99
|
-
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
100
|
-
});
|
|
101
|
-
const minioClient = createMinioClient();
|
|
102
|
-
const bucketName = env.MINIO_BUCKET_NAME ?? "";
|
|
103
|
-
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
104
|
-
const exported = makeExportedStatus();
|
|
105
|
-
const fileIsbns = new Set();
|
|
106
|
-
async function uploadBookImage(localPath) {
|
|
107
|
-
const fileName = basename(localPath);
|
|
108
|
-
const buffer = await readFile(localPath);
|
|
109
|
-
const originalKey = `${ORIGINAL_BOOK_IMAGE_PATH}/${fileName}`;
|
|
110
|
-
try {
|
|
111
|
-
await minioClient.statObject(bucketName, originalKey);
|
|
112
|
-
}
|
|
113
|
-
catch {
|
|
114
|
-
await minioClient.putObject(bucketName, originalKey, buffer);
|
|
115
|
-
console.log(`š¤ MinIO ć¢ćććć¼ćļ¼originalļ¼: ${fileName}`);
|
|
116
|
-
}
|
|
117
|
-
const thumbnailBuffer = await createWebpThumbnail(buffer, {
|
|
118
|
-
width: THUMBNAIL_SIZE,
|
|
119
|
-
height: THUMBNAIL_SIZE,
|
|
120
|
-
});
|
|
121
|
-
const thumbnailKey = `${THUMBNAIL_BOOK_IMAGE_PATH}/${fileName}`;
|
|
122
|
-
await minioClient.putObject(bucketName, thumbnailKey, Buffer.from(thumbnailBuffer));
|
|
123
|
-
return fileName;
|
|
124
|
-
}
|
|
125
|
-
async function resolveNewImagePath(localImagePath, fallback) {
|
|
126
|
-
if (!localImagePath)
|
|
127
|
-
return fallback;
|
|
128
|
-
if (dryRun)
|
|
129
|
-
return basename(localImagePath);
|
|
130
|
-
return uploadBookImage(localImagePath);
|
|
131
|
-
}
|
|
132
|
-
async function ingestBooks() {
|
|
133
|
-
const files = await globPaths(`${contentsPath}/markdown/book/*.md`);
|
|
134
|
-
console.log(`š ${files.length} ä»¶ć®ćć”ć¤ć«ćę¤åŗćć¾ććć`);
|
|
135
|
-
const imageFiles = await globPaths(`${contentsPath}/image/book/*`);
|
|
136
|
-
const bookImageMap = new Map();
|
|
137
|
-
for (const f of imageFiles) {
|
|
138
|
-
if (getContentType(f) === null)
|
|
139
|
-
continue;
|
|
140
|
-
const rawIsbn = basename(f, extname(f));
|
|
141
|
-
try {
|
|
142
|
-
const isbn = makeISBN(rawIsbn);
|
|
143
|
-
bookImageMap.set(isbn, f);
|
|
144
|
-
}
|
|
145
|
-
catch {
|
|
146
|
-
console.error(`ā ļø ē”å¹ćŖē»åISBN: ${basename(f)}`);
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
console.log(`š¼ļø ${bookImageMap.size} ä»¶ć®ęøē±ē»åćę¤åŗćć¾ććć`);
|
|
150
|
-
const existingBooks = await prisma.book.findMany({
|
|
151
|
-
where: { userId },
|
|
152
|
-
select: {
|
|
153
|
-
id: true,
|
|
154
|
-
isbn: true,
|
|
155
|
-
title: true,
|
|
156
|
-
markdown: true,
|
|
157
|
-
imagePath: true,
|
|
158
|
-
rating: true,
|
|
159
|
-
tags: true,
|
|
160
|
-
googleSubTitle: true,
|
|
161
|
-
googleAuthors: true,
|
|
162
|
-
googleDescription: true,
|
|
163
|
-
googleImgSrc: true,
|
|
164
|
-
googleHref: true,
|
|
165
|
-
},
|
|
166
|
-
});
|
|
167
|
-
const existingBooksMap = new Map(existingBooks.map((b) => [b.isbn, b]));
|
|
168
|
-
console.log(`š DB ć« ${existingBooksMap.size} ä»¶ć®ę¢åęøē±ćććć¾ćć`);
|
|
169
|
-
let insertedCount = 0;
|
|
170
|
-
let updatedCount = 0;
|
|
171
|
-
let skippedCount = 0;
|
|
172
|
-
let errorCount = 0;
|
|
173
|
-
async function updateExistingBook(existing, parsed, localImagePath, isbn, title, markdown) {
|
|
174
|
-
const newImagePath = await resolveNewImagePath(localImagePath, existing.imagePath);
|
|
175
|
-
if (dryRun) {
|
|
176
|
-
console.log(`š [dry-run] ę“ę°äŗå®: ${isbn} (${title})`);
|
|
177
|
-
return;
|
|
178
|
-
}
|
|
179
|
-
await prisma.book.update({
|
|
180
|
-
where: { id: existing.id },
|
|
181
|
-
data: {
|
|
182
|
-
title,
|
|
183
|
-
markdown,
|
|
184
|
-
imagePath: newImagePath,
|
|
185
|
-
rating: parsed.rating,
|
|
186
|
-
tags: parsed.tags,
|
|
187
|
-
googleSubTitle: parsed.googleSubTitle,
|
|
188
|
-
googleAuthors: parsed.googleAuthors,
|
|
189
|
-
googleDescription: parsed.googleDescription,
|
|
190
|
-
googleImgSrc: parsed.googleImgSrc,
|
|
191
|
-
googleHref: parsed.googleHref,
|
|
192
|
-
},
|
|
193
|
-
});
|
|
194
|
-
console.log(`š ę“ę°: ${isbn} (${title})`);
|
|
195
|
-
}
|
|
196
|
-
async function createNewBook(parsed, localImagePath, isbn, title, markdown) {
|
|
197
|
-
const newImagePath = await resolveNewImagePath(localImagePath, null);
|
|
198
|
-
if (dryRun) {
|
|
199
|
-
console.log(`š [dry-run] ęæå
„äŗå®: ${isbn} (${title})`);
|
|
200
|
-
return;
|
|
201
|
-
}
|
|
202
|
-
await prisma.book.create({
|
|
203
|
-
data: {
|
|
204
|
-
id: String(makeId()),
|
|
205
|
-
isbn,
|
|
206
|
-
title,
|
|
207
|
-
markdown,
|
|
208
|
-
imagePath: newImagePath,
|
|
209
|
-
status: exported.status,
|
|
210
|
-
exportedAt: exported.exportedAt,
|
|
211
|
-
userId,
|
|
212
|
-
createdAt: new Date(),
|
|
213
|
-
rating: parsed.rating,
|
|
214
|
-
tags: parsed.tags,
|
|
215
|
-
googleSubTitle: parsed.googleSubTitle,
|
|
216
|
-
googleAuthors: parsed.googleAuthors,
|
|
217
|
-
googleDescription: parsed.googleDescription,
|
|
218
|
-
googleImgSrc: parsed.googleImgSrc,
|
|
219
|
-
googleHref: parsed.googleHref,
|
|
220
|
-
},
|
|
221
|
-
});
|
|
222
|
-
console.log(`ā
ęæå
„: ${isbn} (${title})`);
|
|
223
|
-
}
|
|
224
|
-
async function processBookFile(filePath) {
|
|
225
|
-
const rawIsbn = basename(filePath, ".md");
|
|
226
|
-
let isbn = rawIsbn;
|
|
227
|
-
let title = "";
|
|
228
|
-
let markdown = null;
|
|
229
|
-
try {
|
|
230
|
-
isbn = makeISBN(rawIsbn);
|
|
231
|
-
fileIsbns.add(isbn);
|
|
232
|
-
const content = await readFile(filePath, "utf-8");
|
|
233
|
-
const parsed = parseBookFile(content);
|
|
234
|
-
title = parsed.title;
|
|
235
|
-
markdown = parsed.markdown;
|
|
236
|
-
if (!title) {
|
|
237
|
-
console.error(`ā ļø ćæć¤ćć«ćŖć: ${basename(filePath)}`);
|
|
238
|
-
return "error";
|
|
239
|
-
}
|
|
240
|
-
const localImagePath = bookImageMap.get(isbn);
|
|
241
|
-
const existing = existingBooksMap.get(isbn);
|
|
242
|
-
if (existing) {
|
|
243
|
-
const expectedImagePath = localImagePath
|
|
244
|
-
? basename(localImagePath)
|
|
245
|
-
: existing.imagePath;
|
|
246
|
-
if (bookMatchesExisting(existing, parsed, expectedImagePath)) {
|
|
247
|
-
return "skipped";
|
|
248
|
-
}
|
|
249
|
-
await updateExistingBook(existing, parsed, localImagePath, isbn, title, markdown);
|
|
250
|
-
return "updated";
|
|
251
|
-
}
|
|
252
|
-
await createNewBook(parsed, localImagePath, isbn, title, markdown);
|
|
253
|
-
return "inserted";
|
|
254
|
-
}
|
|
255
|
-
catch (error) {
|
|
256
|
-
console.error(`ā ćØć©ć¼ļ¼${basename(filePath)}ļ¼:`, `isbn=${isbn}`, `title(${title.length}ęå)`, `markdown(${(markdown ?? "").length}ęå)`, error);
|
|
257
|
-
return "error";
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
for (const filePath of files) {
|
|
261
|
-
const result = await processBookFile(filePath);
|
|
262
|
-
switch (result) {
|
|
263
|
-
case "inserted":
|
|
264
|
-
insertedCount++;
|
|
265
|
-
break;
|
|
266
|
-
case "updated":
|
|
267
|
-
updatedCount++;
|
|
268
|
-
break;
|
|
269
|
-
case "skipped":
|
|
270
|
-
skippedCount++;
|
|
271
|
-
break;
|
|
272
|
-
case "error":
|
|
273
|
-
errorCount++;
|
|
274
|
-
break;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
return { insertedCount, updatedCount, skippedCount, errorCount };
|
|
278
|
-
}
|
|
279
|
-
async function purgeBooks() {
|
|
280
|
-
const exportedBooks = await prisma.book.findMany({
|
|
281
|
-
where: { userId, status: exported.status },
|
|
282
|
-
select: { id: true, isbn: true },
|
|
283
|
-
});
|
|
284
|
-
const toDelete = exportedBooks.filter((b) => !fileIsbns.has(b.isbn));
|
|
285
|
-
if (toDelete.length === 0) {
|
|
286
|
-
console.log("šļø åé¤åÆ¾č±”ćŖć");
|
|
287
|
-
return 0;
|
|
288
|
-
}
|
|
289
|
-
let deletedCount = 0;
|
|
290
|
-
for (const book of toDelete) {
|
|
291
|
-
if (dryRun) {
|
|
292
|
-
console.log(`šļø [dry-run] åé¤äŗå®: ${book.isbn}`);
|
|
293
|
-
}
|
|
294
|
-
else {
|
|
295
|
-
await prisma.book.delete({ where: { id: book.id } });
|
|
296
|
-
console.log(`šļø åé¤: ${book.isbn}`);
|
|
297
|
-
}
|
|
298
|
-
deletedCount++;
|
|
299
|
-
}
|
|
300
|
-
return deletedCount;
|
|
301
|
-
}
|
|
302
|
-
try {
|
|
303
|
-
const { insertedCount, updatedCount, skippedCount, errorCount } = await ingestBooks();
|
|
304
|
-
const deletedCount = await purgeBooks();
|
|
305
|
-
console.log(`\nš ēµę: ęæå
„ ${insertedCount} ä»¶, ę“ę° ${updatedCount} ä»¶, ć¹ććć ${skippedCount} ä»¶, åé¤ ${deletedCount} ä»¶, ćØć©ć¼ ${errorCount} ä»¶${dryRun ? " (dry-run)" : ""}`);
|
|
306
|
-
await notificationService.notifyInfo(`${SCRIPT_NAME} completed`, {
|
|
307
|
-
caller: SCRIPT_NAME,
|
|
308
|
-
});
|
|
309
|
-
}
|
|
310
|
-
catch (error) {
|
|
311
|
-
console.error("ā ćØć©ć¼ćēŗēćć¾ćć:", error);
|
|
312
|
-
await notificationService.notifyError(`${SCRIPT_NAME} failed: ${String(error)}`, {
|
|
313
|
-
caller: SCRIPT_NAME,
|
|
314
|
-
});
|
|
315
|
-
process.exit(1);
|
|
316
|
-
}
|
|
317
|
-
finally {
|
|
318
|
-
await prisma.$disconnect();
|
|
319
|
-
}
|
|
66
|
+
}
|
|
320
67
|
}
|
|
321
|
-
|
|
322
|
-
console.error(error);
|
|
323
|
-
process.exit(1);
|
|
324
|
-
});
|
|
325
|
-
//# sourceMappingURL=ingest-books.js.map
|
|
68
|
+
await runContentImport("books", source);
|