@s-hirano-ist/s-scripts 1.26.8 โ 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/chunk-6ADTNHCK.js +85 -0
- package/dist/chunk-7PVU2Z32.js +71 -0
- package/dist/chunk-AJMTRCPJ.js +15 -0
- package/dist/chunk-AUD3PD6K.js +85 -0
- package/dist/chunk-F4J6JBR4.js +115 -0
- package/dist/chunk-HDYQV2Y2.js +58 -0
- package/dist/chunk-J4VLH4VA.js +267 -0
- package/dist/chunk-JZOR5GA2.js +42 -0
- package/dist/chunk-KKZP7ZF3.js +77 -0
- package/dist/chunk-M275YZQP.js +7723 -0
- package/dist/chunk-Q6QUV5VR.js +102 -0
- package/dist/chunk-TK456CR2.js +85 -0
- package/dist/chunk-UXGLXHQ6.js +40 -0
- package/dist/chunk-XG6LAGZO.js +590 -0
- package/dist/cleanup-minio-images.js +115 -92
- package/dist/enrich-books.js +87 -78
- package/dist/fetch-articles.js +101 -90
- package/dist/fetch-books.js +147 -128
- package/dist/fetch-images.js +88 -72
- package/dist/fetch-notes.js +66 -54
- package/dist/find-duplicate-json-articles.js +100 -97
- package/dist/fix-book-frontmatter.js +111 -99
- package/dist/fix-image-formats.js +81 -65
- package/dist/ingest-articles.js +30 -249
- package/dist/ingest-books.js +61 -318
- package/dist/ingest-images.js +33 -184
- package/dist/ingest-notes.js +41 -171
- package/dist/rag/delete-collection.js +32 -22
- package/dist/rag/ingest.js +371 -92
- package/dist/rag/search.js +80 -76
- package/dist/reset-articles.js +57 -60
- package/dist/reset-books.js +57 -59
- package/dist/reset-images.js +57 -59
- package/dist/reset-notes.js +57 -59
- package/dist/revert-articles.js +53 -55
- package/dist/revert-books.js +53 -53
- package/dist/revert-images.js +53 -53
- package/dist/revert-notes.js +53 -53
- package/dist/s-content-format.js +103 -13
- package/dist/update-json-articles.js +101 -97
- package/dist/update-raw-articles.js +130 -130
- package/dist/validate-json-articles.js +129 -130
- package/package.json +13 -50
- package/dist/cleanup-minio-images.d.ts +0 -3
- package/dist/cleanup-minio-images.d.ts.map +0 -1
- package/dist/cleanup-minio-images.js.map +0 -1
- package/dist/enrich-books.d.ts +0 -3
- package/dist/enrich-books.d.ts.map +0 -1
- package/dist/enrich-books.js.map +0 -1
- package/dist/fetch-articles.d.ts +0 -3
- package/dist/fetch-articles.d.ts.map +0 -1
- package/dist/fetch-articles.js.map +0 -1
- package/dist/fetch-books.d.ts +0 -3
- package/dist/fetch-books.d.ts.map +0 -1
- package/dist/fetch-books.js.map +0 -1
- package/dist/fetch-images.d.ts +0 -3
- package/dist/fetch-images.d.ts.map +0 -1
- package/dist/fetch-images.js.map +0 -1
- package/dist/fetch-notes.d.ts +0 -3
- package/dist/fetch-notes.d.ts.map +0 -1
- package/dist/fetch-notes.js.map +0 -1
- package/dist/find-duplicate-json-articles.d.ts +0 -3
- package/dist/find-duplicate-json-articles.d.ts.map +0 -1
- package/dist/find-duplicate-json-articles.js.map +0 -1
- package/dist/fix-book-frontmatter.d.ts +0 -3
- package/dist/fix-book-frontmatter.d.ts.map +0 -1
- package/dist/fix-book-frontmatter.js.map +0 -1
- package/dist/fix-image-formats.d.ts +0 -3
- package/dist/fix-image-formats.d.ts.map +0 -1
- package/dist/fix-image-formats.js.map +0 -1
- package/dist/infrastructures/articles-command-repository.d.ts +0 -57
- package/dist/infrastructures/articles-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/articles-command-repository.js +0 -68
- package/dist/infrastructures/articles-command-repository.js.map +0 -1
- package/dist/infrastructures/books-command-repository.d.ts +0 -34
- package/dist/infrastructures/books-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/books-command-repository.js +0 -53
- package/dist/infrastructures/books-command-repository.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.d.ts +0 -9
- package/dist/infrastructures/cache-invalidation-client.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.js +0 -21
- package/dist/infrastructures/cache-invalidation-client.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.d.ts +0 -2
- package/dist/infrastructures/cache-invalidation-client.test.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.js +0 -25
- package/dist/infrastructures/cache-invalidation-client.test.js.map +0 -1
- package/dist/infrastructures/images-command-repository.d.ts +0 -34
- package/dist/infrastructures/images-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/images-command-repository.js +0 -53
- package/dist/infrastructures/images-command-repository.js.map +0 -1
- package/dist/infrastructures/notes-command-repository.d.ts +0 -34
- package/dist/infrastructures/notes-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/notes-command-repository.js +0 -53
- package/dist/infrastructures/notes-command-repository.js.map +0 -1
- package/dist/ingest-articles.d.ts +0 -3
- package/dist/ingest-articles.d.ts.map +0 -1
- package/dist/ingest-articles.js.map +0 -1
- package/dist/ingest-books.d.ts +0 -3
- package/dist/ingest-books.d.ts.map +0 -1
- package/dist/ingest-books.js.map +0 -1
- package/dist/ingest-images.d.ts +0 -3
- package/dist/ingest-images.d.ts.map +0 -1
- package/dist/ingest-images.js.map +0 -1
- package/dist/ingest-notes.d.ts +0 -3
- package/dist/ingest-notes.d.ts.map +0 -1
- package/dist/ingest-notes.js.map +0 -1
- package/dist/lib/glob-paths.d.ts +0 -6
- package/dist/lib/glob-paths.d.ts.map +0 -1
- package/dist/lib/glob-paths.js +0 -9
- package/dist/lib/glob-paths.js.map +0 -1
- package/dist/lib/glob-paths.test.d.ts +0 -2
- package/dist/lib/glob-paths.test.d.ts.map +0 -1
- package/dist/lib/glob-paths.test.js +0 -30
- package/dist/lib/glob-paths.test.js.map +0 -1
- package/dist/lib/html-charset.d.ts +0 -4
- package/dist/lib/html-charset.d.ts.map +0 -1
- package/dist/lib/html-charset.js +0 -33
- package/dist/lib/html-charset.js.map +0 -1
- package/dist/lib/html-charset.test.d.ts +0 -2
- package/dist/lib/html-charset.test.d.ts.map +0 -1
- package/dist/lib/html-charset.test.js +0 -27
- package/dist/lib/html-charset.test.js.map +0 -1
- package/dist/rag/delete-collection.d.ts +0 -3
- package/dist/rag/delete-collection.d.ts.map +0 -1
- package/dist/rag/delete-collection.js.map +0 -1
- package/dist/rag/ingest-config.d.ts +0 -8
- package/dist/rag/ingest-config.d.ts.map +0 -1
- package/dist/rag/ingest-config.js +0 -8
- package/dist/rag/ingest-config.js.map +0 -1
- package/dist/rag/ingest.d.ts +0 -3
- package/dist/rag/ingest.d.ts.map +0 -1
- package/dist/rag/ingest.js.map +0 -1
- package/dist/rag/search.d.ts +0 -3
- package/dist/rag/search.d.ts.map +0 -1
- package/dist/rag/search.js.map +0 -1
- package/dist/reset-articles.d.ts +0 -3
- package/dist/reset-articles.d.ts.map +0 -1
- package/dist/reset-articles.js.map +0 -1
- package/dist/reset-books.d.ts +0 -3
- package/dist/reset-books.d.ts.map +0 -1
- package/dist/reset-books.js.map +0 -1
- package/dist/reset-images.d.ts +0 -3
- package/dist/reset-images.d.ts.map +0 -1
- package/dist/reset-images.js.map +0 -1
- package/dist/reset-notes.d.ts +0 -3
- package/dist/reset-notes.d.ts.map +0 -1
- package/dist/reset-notes.js.map +0 -1
- package/dist/revert-articles.d.ts +0 -3
- package/dist/revert-articles.d.ts.map +0 -1
- package/dist/revert-articles.js.map +0 -1
- package/dist/revert-books.d.ts +0 -3
- package/dist/revert-books.d.ts.map +0 -1
- package/dist/revert-books.js.map +0 -1
- package/dist/revert-images.d.ts +0 -3
- package/dist/revert-images.d.ts.map +0 -1
- package/dist/revert-images.js.map +0 -1
- package/dist/revert-notes.d.ts +0 -3
- package/dist/revert-notes.d.ts.map +0 -1
- package/dist/revert-notes.js.map +0 -1
- package/dist/s-content-format-runner.d.ts +0 -10
- package/dist/s-content-format-runner.d.ts.map +0 -1
- package/dist/s-content-format-runner.js +0 -76
- package/dist/s-content-format-runner.js.map +0 -1
- package/dist/s-content-format.d.ts +0 -3
- package/dist/s-content-format.d.ts.map +0 -1
- package/dist/s-content-format.js.map +0 -1
- package/dist/s-content-format.test.d.ts +0 -2
- package/dist/s-content-format.test.d.ts.map +0 -1
- package/dist/s-content-format.test.js +0 -99
- package/dist/s-content-format.test.js.map +0 -1
- package/dist/update-json-articles.d.ts +0 -3
- package/dist/update-json-articles.d.ts.map +0 -1
- package/dist/update-json-articles.js.map +0 -1
- package/dist/update-raw-articles.d.ts +0 -3
- package/dist/update-raw-articles.d.ts.map +0 -1
- package/dist/update-raw-articles.js.map +0 -1
- package/dist/validate-json-articles.d.ts +0 -3
- package/dist/validate-json-articles.d.ts.map +0 -1
- package/dist/validate-json-articles.js.map +0 -1
|
@@ -1,105 +1,128 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
createMinioClient
|
|
5
|
+
} from "./chunk-F4J6JBR4.js";
|
|
6
|
+
import {
|
|
7
|
+
createPushoverService
|
|
8
|
+
} from "./chunk-7PVU2Z32.js";
|
|
9
|
+
import {
|
|
10
|
+
globPaths
|
|
11
|
+
} from "./chunk-AJMTRCPJ.js";
|
|
12
|
+
import "./chunk-JZOR5GA2.js";
|
|
13
|
+
|
|
14
|
+
// src/cleanup-minio-images.ts
|
|
4
15
|
import { basename, extname } from "node:path";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const THUMBNAIL_IMAGE_PATH = "images/thumbnail";
|
|
16
|
+
var SCRIPT_NAME = "cleanup-minio-images";
|
|
17
|
+
var ORIGINAL_IMAGE_PATH = "images/original";
|
|
18
|
+
var THUMBNAIL_IMAGE_PATH = "images/thumbnail";
|
|
9
19
|
function isImageFile(filePath) {
|
|
10
|
-
|
|
11
|
-
|
|
20
|
+
const ext = extname(filePath).toLowerCase();
|
|
21
|
+
return [".jpg", ".jpeg", ".png", ".webp"].includes(ext);
|
|
12
22
|
}
|
|
13
23
|
function listObjects(client, bucket, prefix) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
return promise;
|
|
24
|
+
const { promise, resolve, reject } = Promise.withResolvers();
|
|
25
|
+
const keys = [];
|
|
26
|
+
const stream = client.listObjectsV2(bucket, prefix, true);
|
|
27
|
+
stream.on("data", (obj) => {
|
|
28
|
+
if (obj.name) keys.push(obj.name);
|
|
29
|
+
});
|
|
30
|
+
stream.on("end", () => resolve(keys));
|
|
31
|
+
stream.on("error", reject);
|
|
32
|
+
return promise;
|
|
24
33
|
}
|
|
25
34
|
async function main() {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
36
|
+
const env = {
|
|
37
|
+
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
38
|
+
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
39
|
+
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
40
|
+
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME
|
|
41
|
+
};
|
|
42
|
+
if (Object.values(env).some((v) => !v)) {
|
|
43
|
+
throw new Error("Required environment variables are not set.");
|
|
44
|
+
}
|
|
45
|
+
if (process.env.MINIO_USE_SSL === "true" && (!process.env.CF_ACCESS_CLIENT_ID || !process.env.CF_ACCESS_CLIENT_SECRET)) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
"CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true."
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
51
|
+
const notificationService = createPushoverService({
|
|
52
|
+
url: env.PUSHOVER_URL ?? "",
|
|
53
|
+
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
54
|
+
appToken: env.PUSHOVER_APP_TOKEN ?? ""
|
|
55
|
+
});
|
|
56
|
+
const minioClient = createMinioClient();
|
|
57
|
+
const bucketName = env.MINIO_BUCKET_NAME ?? "";
|
|
58
|
+
try {
|
|
59
|
+
const localFiles = await globPaths(`${contentsPath}/image/dump/*`);
|
|
60
|
+
const localFileNames = new Set(
|
|
61
|
+
localFiles.filter((f) => isImageFile(f)).map((f) => basename(f))
|
|
62
|
+
);
|
|
63
|
+
console.log(
|
|
64
|
+
`\u{1F4C1} \u30ED\u30FC\u30AB\u30EB\u306B ${localFileNames.size} \u4EF6\u306E\u753B\u50CF\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u51FA\u3057\u307E\u3057\u305F\u3002`
|
|
65
|
+
);
|
|
66
|
+
const [originalKeys, thumbnailKeys] = await Promise.all([
|
|
67
|
+
listObjects(minioClient, bucketName, `${ORIGINAL_IMAGE_PATH}/`),
|
|
68
|
+
listObjects(minioClient, bucketName, `${THUMBNAIL_IMAGE_PATH}/`)
|
|
69
|
+
]);
|
|
70
|
+
console.log(
|
|
71
|
+
`\u{1F4E6} MinIO: original ${originalKeys.length} \u4EF6, thumbnail ${thumbnailKeys.length} \u4EF6`
|
|
72
|
+
);
|
|
73
|
+
const orphanedOriginals = originalKeys.filter(
|
|
74
|
+
(key) => !localFileNames.has(basename(key))
|
|
75
|
+
);
|
|
76
|
+
const orphanedThumbnails = thumbnailKeys.filter(
|
|
77
|
+
(key) => !localFileNames.has(basename(key))
|
|
78
|
+
);
|
|
79
|
+
const totalOrphaned = orphanedOriginals.length + orphanedThumbnails.length;
|
|
80
|
+
if (totalOrphaned === 0) {
|
|
81
|
+
console.log("\u2705 \u5B64\u7ACB\u30D5\u30A1\u30A4\u30EB\u306A\u3057");
|
|
82
|
+
await notificationService.notifyInfo(
|
|
83
|
+
`${SCRIPT_NAME} completed: no orphaned files`,
|
|
84
|
+
{ caller: SCRIPT_NAME }
|
|
85
|
+
);
|
|
86
|
+
return;
|
|
39
87
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
console.log(`๐ ใญใผใซใซใซ ${localFileNames.size} ไปถใฎ็ปๅใใกใคใซใๆคๅบใใพใใใ`);
|
|
53
|
-
// 2. MinIOใชใใธใงใฏใไธ่ฆงๅๅพ
|
|
54
|
-
const [originalKeys, thumbnailKeys] = await Promise.all([
|
|
55
|
-
listObjects(minioClient, bucketName, `${ORIGINAL_IMAGE_PATH}/`),
|
|
56
|
-
listObjects(minioClient, bucketName, `${THUMBNAIL_IMAGE_PATH}/`),
|
|
57
|
-
]);
|
|
58
|
-
console.log(`๐ฆ MinIO: original ${originalKeys.length} ไปถ, thumbnail ${thumbnailKeys.length} ไปถ`);
|
|
59
|
-
// 3. ๅทฎๅๆคๅบ
|
|
60
|
-
const orphanedOriginals = originalKeys.filter((key) => !localFileNames.has(basename(key)));
|
|
61
|
-
const orphanedThumbnails = thumbnailKeys.filter((key) => !localFileNames.has(basename(key)));
|
|
62
|
-
const totalOrphaned = orphanedOriginals.length + orphanedThumbnails.length;
|
|
63
|
-
if (totalOrphaned === 0) {
|
|
64
|
-
console.log("โ
ๅญค็ซใใกใคใซใชใ");
|
|
65
|
-
await notificationService.notifyInfo(`${SCRIPT_NAME} completed: no orphaned files`, { caller: SCRIPT_NAME });
|
|
66
|
-
return;
|
|
67
|
-
}
|
|
68
|
-
console.log(`๐๏ธ ๅญค็ซใใกใคใซๆคๅบ: original ${orphanedOriginals.length} ไปถ, thumbnail ${orphanedThumbnails.length} ไปถ`);
|
|
69
|
-
// 4. ๅ้คๅฎ่ก
|
|
70
|
-
let deletedCount = 0;
|
|
71
|
-
let errorCount = 0;
|
|
72
|
-
for (const key of [...orphanedOriginals, ...orphanedThumbnails]) {
|
|
73
|
-
try {
|
|
74
|
-
if (dryRun) {
|
|
75
|
-
console.log(`๐ [dry-run] ๅ้คไบๅฎ: ${key}`);
|
|
76
|
-
}
|
|
77
|
-
else {
|
|
78
|
-
await minioClient.removeObject(bucketName, key);
|
|
79
|
-
console.log(`๐๏ธ ๅ้ค: ${key}`);
|
|
80
|
-
}
|
|
81
|
-
deletedCount++;
|
|
82
|
-
}
|
|
83
|
-
catch (error) {
|
|
84
|
-
console.error(`โ ๅ้คใจใฉใผ๏ผ${key}๏ผ:`, error);
|
|
85
|
-
errorCount++;
|
|
86
|
-
}
|
|
88
|
+
console.log(
|
|
89
|
+
`\u{1F5D1}\uFE0F \u5B64\u7ACB\u30D5\u30A1\u30A4\u30EB\u691C\u51FA: original ${orphanedOriginals.length} \u4EF6, thumbnail ${orphanedThumbnails.length} \u4EF6`
|
|
90
|
+
);
|
|
91
|
+
let deletedCount = 0;
|
|
92
|
+
let errorCount = 0;
|
|
93
|
+
for (const key of [...orphanedOriginals, ...orphanedThumbnails]) {
|
|
94
|
+
try {
|
|
95
|
+
if (dryRun) {
|
|
96
|
+
console.log(`\u{1F50D} [dry-run] \u524A\u9664\u4E88\u5B9A: ${key}`);
|
|
97
|
+
} else {
|
|
98
|
+
await minioClient.removeObject(bucketName, key);
|
|
99
|
+
console.log(`\u{1F5D1}\uFE0F \u524A\u9664: ${key}`);
|
|
87
100
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
catch (error) {
|
|
94
|
-
console.error("โ ใจใฉใผใ็บ็ใใพใใ:", error);
|
|
95
|
-
await notificationService.notifyError(`${SCRIPT_NAME} failed: ${String(error)}`, {
|
|
96
|
-
caller: SCRIPT_NAME,
|
|
97
|
-
});
|
|
98
|
-
process.exit(1);
|
|
101
|
+
deletedCount++;
|
|
102
|
+
} catch (error) {
|
|
103
|
+
console.error(`\u274C \u524A\u9664\u30A8\u30E9\u30FC\uFF08${key}\uFF09:`, error);
|
|
104
|
+
errorCount++;
|
|
105
|
+
}
|
|
99
106
|
}
|
|
107
|
+
console.log(
|
|
108
|
+
`
|
|
109
|
+
\u{1F4CA} \u7D50\u679C: \u524A\u9664 ${deletedCount} \u4EF6, \u30A8\u30E9\u30FC ${errorCount} \u4EF6${dryRun ? " (dry-run)" : ""}`
|
|
110
|
+
);
|
|
111
|
+
await notificationService.notifyInfo(`${SCRIPT_NAME} completed`, {
|
|
112
|
+
caller: SCRIPT_NAME
|
|
113
|
+
});
|
|
114
|
+
} catch (error) {
|
|
115
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
116
|
+
await notificationService.notifyError(
|
|
117
|
+
`${SCRIPT_NAME} failed: ${String(error)}`,
|
|
118
|
+
{
|
|
119
|
+
caller: SCRIPT_NAME
|
|
120
|
+
}
|
|
121
|
+
);
|
|
122
|
+
process.exit(1);
|
|
123
|
+
}
|
|
100
124
|
}
|
|
101
125
|
main().catch((error) => {
|
|
102
|
-
|
|
103
|
-
|
|
126
|
+
console.error(error);
|
|
127
|
+
process.exit(1);
|
|
104
128
|
});
|
|
105
|
-
//# sourceMappingURL=cleanup-minio-images.js.map
|
package/dist/enrich-books.js
CHANGED
|
@@ -1,91 +1,100 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
parseMarkdownFrontmatter
|
|
5
|
+
} from "./chunk-HDYQV2Y2.js";
|
|
6
|
+
import {
|
|
7
|
+
globPaths
|
|
8
|
+
} from "./chunk-AJMTRCPJ.js";
|
|
9
|
+
import "./chunk-JZOR5GA2.js";
|
|
10
|
+
|
|
11
|
+
// src/enrich-books.ts
|
|
2
12
|
import { books as googleBooksApis } from "@googleapis/books";
|
|
3
|
-
import { parseMarkdownFrontmatter } from "@s-hirano-ist/s-search/frontmatter";
|
|
4
13
|
import { dump } from "js-yaml";
|
|
5
|
-
/**
|
|
6
|
-
* markdown/book/*.md ใฎ frontmatter ใซใคใใฆใๆฏๅ Google Books API ใๅผใณๅบใใฆ
|
|
7
|
-
* google* ใใใณ title ใๆๆฐ็ถๆ
ใซๆดๆฐใใในใฏใชใใใ
|
|
8
|
-
*
|
|
9
|
-
* API ใ title ใ่ฟใใๅ ดๅใฏ frontmatter.title ใไธๆธใใใใ
|
|
10
|
-
* notFound (API ใ volume ใ่ฟใใชใ) ใฎๅ ดๅใฏๆๅๅ
ฅๅ title ใไฟๆใใใ
|
|
11
|
-
* markdown ๆฌๆ (h1 ๅซใ) ใซใฏ่งฆใใชใใใใ่ชๅทฑๅ
ฅๅใฎ h1 ใฏใใฎใพใพ็ถญๆใใใใ
|
|
12
|
-
*/
|
|
13
14
|
import { readFile, writeFile } from "node:fs/promises";
|
|
14
15
|
import { basename, join } from "node:path";
|
|
15
16
|
import { setTimeout } from "node:timers/promises";
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const API_INTERVAL_MS = 600;
|
|
17
|
+
var NO_IMG_SRC = "https://s-hirano.com/notFound.png";
|
|
18
|
+
var NOT_FOUND_HREF = "https://s-hirano.com/404";
|
|
19
|
+
var API_INTERVAL_MS = 600;
|
|
20
20
|
function dumpFrontmatter(data) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
return dump(data, {
|
|
22
|
+
lineWidth: -1,
|
|
23
|
+
forceQuotes: false,
|
|
24
|
+
noRefs: true
|
|
25
|
+
});
|
|
26
26
|
}
|
|
27
27
|
async function main() {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
29
|
+
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
30
|
+
const bookDir = join(contentsPath, "markdown", "book");
|
|
31
|
+
const apiKey = process.env.GOOGLE_BOOKS_API_KEY;
|
|
32
|
+
if (!apiKey) {
|
|
33
|
+
throw new Error("GOOGLE_BOOKS_API_KEY is not set");
|
|
34
|
+
}
|
|
35
|
+
const api = googleBooksApis({ version: "v1", auth: apiKey });
|
|
36
|
+
const files = await globPaths(`${bookDir}/*.md`);
|
|
37
|
+
console.log(
|
|
38
|
+
`\u{1F4C1} ${files.length} \u4EF6\u306E markdown \u3092\u691C\u51FA\u3057\u307E\u3057\u305F\u3002${dryRun ? " (dry-run)" : ""}`
|
|
39
|
+
);
|
|
40
|
+
let enrichedCount = 0;
|
|
41
|
+
let errorCount = 0;
|
|
42
|
+
for (const filePath of files.toSorted()) {
|
|
43
|
+
const fileName = basename(filePath);
|
|
44
|
+
const isbn = basename(filePath, ".md");
|
|
45
|
+
try {
|
|
46
|
+
const raw = await readFile(filePath, "utf8");
|
|
47
|
+
const parsed = parseMarkdownFrontmatter(raw);
|
|
48
|
+
const data = parsed.data;
|
|
49
|
+
console.log(`\u{1F50D} ${fileName}: Google Books API \u554F\u3044\u5408\u308F\u305B\u4E2D...`);
|
|
50
|
+
const book = await api.volumes.list({ q: `isbn:${isbn}` });
|
|
51
|
+
await setTimeout(API_INTERVAL_MS);
|
|
52
|
+
if (book.status !== 200) {
|
|
53
|
+
throw new Error(`Google Books API status ${book.status}`);
|
|
54
|
+
}
|
|
55
|
+
const volume = book.data.items?.[0]?.volumeInfo;
|
|
56
|
+
const httpsImgSrc = (volume?.imageLinks?.thumbnail ?? NO_IMG_SRC).replace(
|
|
57
|
+
"http://",
|
|
58
|
+
"https://"
|
|
59
|
+
);
|
|
60
|
+
const httpsHref = (volume?.infoLink ?? NOT_FOUND_HREF).replace(
|
|
61
|
+
"http://",
|
|
62
|
+
"https://"
|
|
63
|
+
);
|
|
64
|
+
const next = {
|
|
65
|
+
...data,
|
|
66
|
+
googleSubtitle: volume?.subtitle ?? "",
|
|
67
|
+
googleAuthors: volume?.authors ?? [],
|
|
68
|
+
googleDescription: volume?.description ?? "No description",
|
|
69
|
+
googleImgSrc: httpsImgSrc,
|
|
70
|
+
googleHref: httpsHref
|
|
71
|
+
};
|
|
72
|
+
if (typeof volume?.title === "string" && volume.title !== "") {
|
|
73
|
+
next.title = volume.title;
|
|
74
|
+
}
|
|
75
|
+
const body = parsed.content.replace(/^\n+/u, "");
|
|
76
|
+
const newContent = `---
|
|
77
|
+
${dumpFrontmatter(next)}---
|
|
78
|
+
|
|
79
|
+
${body}`;
|
|
80
|
+
if (dryRun) {
|
|
81
|
+
console.log(" [dry-run] \u66F8\u304D\u8FBC\u307F\u30B9\u30AD\u30C3\u30D7");
|
|
82
|
+
} else {
|
|
83
|
+
await writeFile(filePath, newContent, "utf8");
|
|
84
|
+
console.log(` \u2705 ${volume?.title ?? "(title not found)"}`);
|
|
85
|
+
}
|
|
86
|
+
enrichedCount++;
|
|
87
|
+
} catch (error) {
|
|
88
|
+
console.error(`\u274C ${fileName}:`, error);
|
|
89
|
+
errorCount++;
|
|
34
90
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
for (const filePath of files.toSorted()) {
|
|
41
|
-
const fileName = basename(filePath);
|
|
42
|
-
const isbn = basename(filePath, ".md");
|
|
43
|
-
try {
|
|
44
|
-
const raw = await readFile(filePath, "utf8");
|
|
45
|
-
const parsed = parseMarkdownFrontmatter(raw);
|
|
46
|
-
const data = parsed.data;
|
|
47
|
-
console.log(`๐ ${fileName}: Google Books API ๅใๅใใไธญ...`);
|
|
48
|
-
const book = await api.volumes.list({ q: `isbn:${isbn}` });
|
|
49
|
-
await setTimeout(API_INTERVAL_MS);
|
|
50
|
-
if (book.status !== 200) {
|
|
51
|
-
throw new Error(`Google Books API status ${book.status}`);
|
|
52
|
-
}
|
|
53
|
-
const volume = book.data.items?.[0]?.volumeInfo;
|
|
54
|
-
const httpsImgSrc = (volume?.imageLinks?.thumbnail ?? NO_IMG_SRC).replace("http://", "https://");
|
|
55
|
-
const httpsHref = (volume?.infoLink ?? NOT_FOUND_HREF).replace("http://", "https://");
|
|
56
|
-
const next = {
|
|
57
|
-
...data,
|
|
58
|
-
googleSubtitle: volume?.subtitle ?? "",
|
|
59
|
-
googleAuthors: volume?.authors ?? [],
|
|
60
|
-
googleDescription: volume?.description ?? "No description",
|
|
61
|
-
googleImgSrc: httpsImgSrc,
|
|
62
|
-
googleHref: httpsHref,
|
|
63
|
-
};
|
|
64
|
-
// API ใ title ใ่ฟใใๅ ดๅใฎใฟ frontmatter.title ใไธๆธใใ
|
|
65
|
-
// notFound ใฎๅ ดๅใฏๆๅๅ
ฅๅ title ใไฟๆใใใ
|
|
66
|
-
if (typeof volume?.title === "string" && volume.title !== "") {
|
|
67
|
-
next.title = volume.title;
|
|
68
|
-
}
|
|
69
|
-
const body = parsed.content.replace(/^\n+/u, "");
|
|
70
|
-
const newContent = `---\n${dumpFrontmatter(next)}---\n\n${body}`;
|
|
71
|
-
if (dryRun) {
|
|
72
|
-
console.log(" [dry-run] ๆธใ่พผใฟในใญใใ");
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
await writeFile(filePath, newContent, "utf8");
|
|
76
|
-
console.log(` โ
${volume?.title ?? "(title not found)"}`);
|
|
77
|
-
}
|
|
78
|
-
enrichedCount++;
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
console.error(`โ ${fileName}:`, error);
|
|
82
|
-
errorCount++;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
console.log(`\n๐ ็ตๆ: ๆดๆฐ ${enrichedCount} ไปถ, ใจใฉใผ ${errorCount} ไปถ${dryRun ? " (dry-run)" : ""}`);
|
|
91
|
+
}
|
|
92
|
+
console.log(
|
|
93
|
+
`
|
|
94
|
+
\u{1F4CA} \u7D50\u679C: \u66F4\u65B0 ${enrichedCount} \u4EF6, \u30A8\u30E9\u30FC ${errorCount} \u4EF6${dryRun ? " (dry-run)" : ""}`
|
|
95
|
+
);
|
|
86
96
|
}
|
|
87
97
|
main().catch((error) => {
|
|
88
|
-
|
|
89
|
-
|
|
98
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
99
|
+
process.exit(1);
|
|
90
100
|
});
|
|
91
|
-
//# sourceMappingURL=enrich-books.js.map
|
package/dist/fetch-articles.js
CHANGED
|
@@ -1,104 +1,115 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
createPrismaClient,
|
|
5
|
+
makeUnexportedStatus,
|
|
6
|
+
makeUserId
|
|
7
|
+
} from "./chunk-M275YZQP.js";
|
|
8
|
+
import {
|
|
9
|
+
createPushoverService
|
|
10
|
+
} from "./chunk-7PVU2Z32.js";
|
|
11
|
+
import "./chunk-JZOR5GA2.js";
|
|
12
|
+
|
|
13
|
+
// src/fetch-articles.ts
|
|
5
14
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
6
15
|
import { dirname } from "node:path";
|
|
7
|
-
|
|
16
|
+
var OUTPUT_PATH = "json/article";
|
|
8
17
|
function categorizeArticles(articles) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
return acc;
|
|
16
|
-
}, {});
|
|
18
|
+
return articles.reduce((acc, d) => {
|
|
19
|
+
const { title, quote, url, categoryName } = d;
|
|
20
|
+
if (!acc[categoryName]) acc[categoryName] = [];
|
|
21
|
+
acc[categoryName].push({ title, quote: quote ?? "", url });
|
|
22
|
+
return acc;
|
|
23
|
+
}, {});
|
|
17
24
|
}
|
|
18
25
|
async function readFileOrCreate(key) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return data;
|
|
31
|
-
}
|
|
26
|
+
const filePath = `${OUTPUT_PATH}/${key}.json`;
|
|
27
|
+
try {
|
|
28
|
+
const data = await readFile(filePath, "utf8");
|
|
29
|
+
return JSON.parse(data);
|
|
30
|
+
} catch {
|
|
31
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
32
|
+
const data = { heading: key, description: "FIXME", body: [] };
|
|
33
|
+
const jsonData = JSON.stringify(data, null, 2);
|
|
34
|
+
await writeFile(filePath, jsonData);
|
|
35
|
+
return data;
|
|
36
|
+
}
|
|
32
37
|
}
|
|
33
38
|
async function exportData(data) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
for (const [key, value] of Object.entries(data)) {
|
|
40
|
+
console.log(`Key: ${key}`);
|
|
41
|
+
const originalData = await readFileOrCreate(key);
|
|
42
|
+
originalData.body.push(...value);
|
|
43
|
+
await writeFile(
|
|
44
|
+
`${OUTPUT_PATH}/${key}.json`,
|
|
45
|
+
`${JSON.stringify(originalData, null, 2)}
|
|
46
|
+
`
|
|
47
|
+
);
|
|
48
|
+
}
|
|
40
49
|
}
|
|
41
50
|
async function main() {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
51
|
+
const env = {
|
|
52
|
+
DATABASE_URL: process.env.DATABASE_URL,
|
|
53
|
+
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
54
|
+
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
55
|
+
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
56
|
+
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT
|
|
57
|
+
};
|
|
58
|
+
if (Object.values(env).some((v) => !v)) {
|
|
59
|
+
throw new Error("Required environment variables are not set.");
|
|
60
|
+
}
|
|
61
|
+
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
62
|
+
const notificationService = createPushoverService({
|
|
63
|
+
url: env.PUSHOVER_URL ?? "",
|
|
64
|
+
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
65
|
+
appToken: env.PUSHOVER_APP_TOKEN ?? ""
|
|
66
|
+
});
|
|
67
|
+
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
68
|
+
const UNEXPORTED = makeUnexportedStatus();
|
|
69
|
+
async function fetchArticles() {
|
|
70
|
+
const rawArticles = await prisma.article.findMany({
|
|
71
|
+
where: { userId, status: UNEXPORTED },
|
|
72
|
+
select: {
|
|
73
|
+
id: true,
|
|
74
|
+
title: true,
|
|
75
|
+
quote: true,
|
|
76
|
+
url: true,
|
|
77
|
+
Category: true
|
|
78
|
+
}
|
|
57
79
|
});
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
catch (error) {
|
|
90
|
-
console.error("โ ใจใฉใผใ็บ็ใใพใใ:", error);
|
|
91
|
-
await notificationService.notifyError(`fetch-articles failed: ${String(error)}`, {
|
|
92
|
-
caller: "fetch-articles",
|
|
93
|
-
});
|
|
94
|
-
process.exit(1);
|
|
95
|
-
}
|
|
96
|
-
finally {
|
|
97
|
-
await prisma.$disconnect();
|
|
98
|
-
}
|
|
80
|
+
const articles = rawArticles.map(
|
|
81
|
+
(d) => ({
|
|
82
|
+
id: d.id,
|
|
83
|
+
title: d.title,
|
|
84
|
+
url: d.url,
|
|
85
|
+
quote: d.quote,
|
|
86
|
+
// oxlint-disable-next-line typescript/no-unnecessary-condition -- Prisma row is an external runtime boundary; relation/name may differ from declared non-null type
|
|
87
|
+
categoryName: d.Category?.name ?? ""
|
|
88
|
+
})
|
|
89
|
+
);
|
|
90
|
+
console.log(`\u{1F4CA} ${articles.length} \u4EF6\u306E\u30C7\u30FC\u30BF\u3092\u53D6\u5F97\u3057\u307E\u3057\u305F\u3002`);
|
|
91
|
+
await exportData(categorizeArticles(articles));
|
|
92
|
+
console.log("\u{1F4BE} \u30C7\u30FC\u30BF\u304CJSON\u30D5\u30A1\u30A4\u30EB\u306B\u66F8\u304D\u51FA\u3055\u308C\u307E\u3057\u305F\u3002");
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
await fetchArticles();
|
|
96
|
+
await notificationService.notifyInfo("fetch-articles completed", {
|
|
97
|
+
caller: "fetch-articles"
|
|
98
|
+
});
|
|
99
|
+
} catch (error) {
|
|
100
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
101
|
+
await notificationService.notifyError(
|
|
102
|
+
`fetch-articles failed: ${String(error)}`,
|
|
103
|
+
{
|
|
104
|
+
caller: "fetch-articles"
|
|
105
|
+
}
|
|
106
|
+
);
|
|
107
|
+
process.exit(1);
|
|
108
|
+
} finally {
|
|
109
|
+
await prisma.$disconnect();
|
|
110
|
+
}
|
|
99
111
|
}
|
|
100
112
|
main().catch((error) => {
|
|
101
|
-
|
|
102
|
-
|
|
113
|
+
console.error(error);
|
|
114
|
+
process.exit(1);
|
|
103
115
|
});
|
|
104
|
-
//# sourceMappingURL=fetch-articles.js.map
|