@s-hirano-ist/s-scripts 1.26.8 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +2 -0
  2. package/dist/chunk-6ADTNHCK.js +85 -0
  3. package/dist/chunk-7PVU2Z32.js +71 -0
  4. package/dist/chunk-AJMTRCPJ.js +15 -0
  5. package/dist/chunk-AUD3PD6K.js +85 -0
  6. package/dist/chunk-F4J6JBR4.js +115 -0
  7. package/dist/chunk-HDYQV2Y2.js +58 -0
  8. package/dist/chunk-J4VLH4VA.js +267 -0
  9. package/dist/chunk-JZOR5GA2.js +42 -0
  10. package/dist/chunk-KKZP7ZF3.js +77 -0
  11. package/dist/chunk-M275YZQP.js +7723 -0
  12. package/dist/chunk-Q6QUV5VR.js +102 -0
  13. package/dist/chunk-TK456CR2.js +85 -0
  14. package/dist/chunk-UXGLXHQ6.js +40 -0
  15. package/dist/chunk-XG6LAGZO.js +590 -0
  16. package/dist/cleanup-minio-images.js +115 -92
  17. package/dist/enrich-books.js +87 -78
  18. package/dist/fetch-articles.js +101 -90
  19. package/dist/fetch-books.js +147 -128
  20. package/dist/fetch-images.js +88 -72
  21. package/dist/fetch-notes.js +66 -54
  22. package/dist/find-duplicate-json-articles.js +100 -97
  23. package/dist/fix-book-frontmatter.js +111 -99
  24. package/dist/fix-image-formats.js +81 -65
  25. package/dist/ingest-articles.js +30 -249
  26. package/dist/ingest-books.js +61 -318
  27. package/dist/ingest-images.js +33 -184
  28. package/dist/ingest-notes.js +41 -171
  29. package/dist/rag/delete-collection.js +32 -22
  30. package/dist/rag/ingest.js +371 -92
  31. package/dist/rag/search.js +80 -76
  32. package/dist/reset-articles.js +57 -60
  33. package/dist/reset-books.js +57 -59
  34. package/dist/reset-images.js +57 -59
  35. package/dist/reset-notes.js +57 -59
  36. package/dist/revert-articles.js +53 -55
  37. package/dist/revert-books.js +53 -53
  38. package/dist/revert-images.js +53 -53
  39. package/dist/revert-notes.js +53 -53
  40. package/dist/s-content-format.js +103 -13
  41. package/dist/update-json-articles.js +101 -97
  42. package/dist/update-raw-articles.js +130 -130
  43. package/dist/validate-json-articles.js +129 -130
  44. package/package.json +13 -50
  45. package/dist/cleanup-minio-images.d.ts +0 -3
  46. package/dist/cleanup-minio-images.d.ts.map +0 -1
  47. package/dist/cleanup-minio-images.js.map +0 -1
  48. package/dist/enrich-books.d.ts +0 -3
  49. package/dist/enrich-books.d.ts.map +0 -1
  50. package/dist/enrich-books.js.map +0 -1
  51. package/dist/fetch-articles.d.ts +0 -3
  52. package/dist/fetch-articles.d.ts.map +0 -1
  53. package/dist/fetch-articles.js.map +0 -1
  54. package/dist/fetch-books.d.ts +0 -3
  55. package/dist/fetch-books.d.ts.map +0 -1
  56. package/dist/fetch-books.js.map +0 -1
  57. package/dist/fetch-images.d.ts +0 -3
  58. package/dist/fetch-images.d.ts.map +0 -1
  59. package/dist/fetch-images.js.map +0 -1
  60. package/dist/fetch-notes.d.ts +0 -3
  61. package/dist/fetch-notes.d.ts.map +0 -1
  62. package/dist/fetch-notes.js.map +0 -1
  63. package/dist/find-duplicate-json-articles.d.ts +0 -3
  64. package/dist/find-duplicate-json-articles.d.ts.map +0 -1
  65. package/dist/find-duplicate-json-articles.js.map +0 -1
  66. package/dist/fix-book-frontmatter.d.ts +0 -3
  67. package/dist/fix-book-frontmatter.d.ts.map +0 -1
  68. package/dist/fix-book-frontmatter.js.map +0 -1
  69. package/dist/fix-image-formats.d.ts +0 -3
  70. package/dist/fix-image-formats.d.ts.map +0 -1
  71. package/dist/fix-image-formats.js.map +0 -1
  72. package/dist/infrastructures/articles-command-repository.d.ts +0 -57
  73. package/dist/infrastructures/articles-command-repository.d.ts.map +0 -1
  74. package/dist/infrastructures/articles-command-repository.js +0 -68
  75. package/dist/infrastructures/articles-command-repository.js.map +0 -1
  76. package/dist/infrastructures/books-command-repository.d.ts +0 -34
  77. package/dist/infrastructures/books-command-repository.d.ts.map +0 -1
  78. package/dist/infrastructures/books-command-repository.js +0 -53
  79. package/dist/infrastructures/books-command-repository.js.map +0 -1
  80. package/dist/infrastructures/cache-invalidation-client.d.ts +0 -9
  81. package/dist/infrastructures/cache-invalidation-client.d.ts.map +0 -1
  82. package/dist/infrastructures/cache-invalidation-client.js +0 -21
  83. package/dist/infrastructures/cache-invalidation-client.js.map +0 -1
  84. package/dist/infrastructures/cache-invalidation-client.test.d.ts +0 -2
  85. package/dist/infrastructures/cache-invalidation-client.test.d.ts.map +0 -1
  86. package/dist/infrastructures/cache-invalidation-client.test.js +0 -25
  87. package/dist/infrastructures/cache-invalidation-client.test.js.map +0 -1
  88. package/dist/infrastructures/images-command-repository.d.ts +0 -34
  89. package/dist/infrastructures/images-command-repository.d.ts.map +0 -1
  90. package/dist/infrastructures/images-command-repository.js +0 -53
  91. package/dist/infrastructures/images-command-repository.js.map +0 -1
  92. package/dist/infrastructures/notes-command-repository.d.ts +0 -34
  93. package/dist/infrastructures/notes-command-repository.d.ts.map +0 -1
  94. package/dist/infrastructures/notes-command-repository.js +0 -53
  95. package/dist/infrastructures/notes-command-repository.js.map +0 -1
  96. package/dist/ingest-articles.d.ts +0 -3
  97. package/dist/ingest-articles.d.ts.map +0 -1
  98. package/dist/ingest-articles.js.map +0 -1
  99. package/dist/ingest-books.d.ts +0 -3
  100. package/dist/ingest-books.d.ts.map +0 -1
  101. package/dist/ingest-books.js.map +0 -1
  102. package/dist/ingest-images.d.ts +0 -3
  103. package/dist/ingest-images.d.ts.map +0 -1
  104. package/dist/ingest-images.js.map +0 -1
  105. package/dist/ingest-notes.d.ts +0 -3
  106. package/dist/ingest-notes.d.ts.map +0 -1
  107. package/dist/ingest-notes.js.map +0 -1
  108. package/dist/lib/glob-paths.d.ts +0 -6
  109. package/dist/lib/glob-paths.d.ts.map +0 -1
  110. package/dist/lib/glob-paths.js +0 -9
  111. package/dist/lib/glob-paths.js.map +0 -1
  112. package/dist/lib/glob-paths.test.d.ts +0 -2
  113. package/dist/lib/glob-paths.test.d.ts.map +0 -1
  114. package/dist/lib/glob-paths.test.js +0 -30
  115. package/dist/lib/glob-paths.test.js.map +0 -1
  116. package/dist/lib/html-charset.d.ts +0 -4
  117. package/dist/lib/html-charset.d.ts.map +0 -1
  118. package/dist/lib/html-charset.js +0 -33
  119. package/dist/lib/html-charset.js.map +0 -1
  120. package/dist/lib/html-charset.test.d.ts +0 -2
  121. package/dist/lib/html-charset.test.d.ts.map +0 -1
  122. package/dist/lib/html-charset.test.js +0 -27
  123. package/dist/lib/html-charset.test.js.map +0 -1
  124. package/dist/rag/delete-collection.d.ts +0 -3
  125. package/dist/rag/delete-collection.d.ts.map +0 -1
  126. package/dist/rag/delete-collection.js.map +0 -1
  127. package/dist/rag/ingest-config.d.ts +0 -8
  128. package/dist/rag/ingest-config.d.ts.map +0 -1
  129. package/dist/rag/ingest-config.js +0 -8
  130. package/dist/rag/ingest-config.js.map +0 -1
  131. package/dist/rag/ingest.d.ts +0 -3
  132. package/dist/rag/ingest.d.ts.map +0 -1
  133. package/dist/rag/ingest.js.map +0 -1
  134. package/dist/rag/search.d.ts +0 -3
  135. package/dist/rag/search.d.ts.map +0 -1
  136. package/dist/rag/search.js.map +0 -1
  137. package/dist/reset-articles.d.ts +0 -3
  138. package/dist/reset-articles.d.ts.map +0 -1
  139. package/dist/reset-articles.js.map +0 -1
  140. package/dist/reset-books.d.ts +0 -3
  141. package/dist/reset-books.d.ts.map +0 -1
  142. package/dist/reset-books.js.map +0 -1
  143. package/dist/reset-images.d.ts +0 -3
  144. package/dist/reset-images.d.ts.map +0 -1
  145. package/dist/reset-images.js.map +0 -1
  146. package/dist/reset-notes.d.ts +0 -3
  147. package/dist/reset-notes.d.ts.map +0 -1
  148. package/dist/reset-notes.js.map +0 -1
  149. package/dist/revert-articles.d.ts +0 -3
  150. package/dist/revert-articles.d.ts.map +0 -1
  151. package/dist/revert-articles.js.map +0 -1
  152. package/dist/revert-books.d.ts +0 -3
  153. package/dist/revert-books.d.ts.map +0 -1
  154. package/dist/revert-books.js.map +0 -1
  155. package/dist/revert-images.d.ts +0 -3
  156. package/dist/revert-images.d.ts.map +0 -1
  157. package/dist/revert-images.js.map +0 -1
  158. package/dist/revert-notes.d.ts +0 -3
  159. package/dist/revert-notes.d.ts.map +0 -1
  160. package/dist/revert-notes.js.map +0 -1
  161. package/dist/s-content-format-runner.d.ts +0 -10
  162. package/dist/s-content-format-runner.d.ts.map +0 -1
  163. package/dist/s-content-format-runner.js +0 -76
  164. package/dist/s-content-format-runner.js.map +0 -1
  165. package/dist/s-content-format.d.ts +0 -3
  166. package/dist/s-content-format.d.ts.map +0 -1
  167. package/dist/s-content-format.js.map +0 -1
  168. package/dist/s-content-format.test.d.ts +0 -2
  169. package/dist/s-content-format.test.d.ts.map +0 -1
  170. package/dist/s-content-format.test.js +0 -99
  171. package/dist/s-content-format.test.js.map +0 -1
  172. package/dist/update-json-articles.d.ts +0 -3
  173. package/dist/update-json-articles.d.ts.map +0 -1
  174. package/dist/update-json-articles.js.map +0 -1
  175. package/dist/update-raw-articles.d.ts +0 -3
  176. package/dist/update-raw-articles.d.ts.map +0 -1
  177. package/dist/update-raw-articles.js.map +0 -1
  178. package/dist/validate-json-articles.d.ts +0 -3
  179. package/dist/validate-json-articles.d.ts.map +0 -1
  180. package/dist/validate-json-articles.js.map +0 -1
@@ -1,252 +1,33 @@
1
1
  #!/usr/bin/env node
2
- import { makeExportedStatus, makeId, makeUserId, } from "@s-hirano-ist/s-core/shared-kernel/entities/common-entity";
3
- import { createPrismaClient } from "@s-hirano-ist/s-database";
4
- import { createPushoverService } from "@s-hirano-ist/s-notification";
2
+ import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
3
+ import {
4
+ runContentImport
5
+ } from "./chunk-XG6LAGZO.js";
6
+ import "./chunk-KKZP7ZF3.js";
7
+ import "./chunk-F4J6JBR4.js";
8
+ import "./chunk-M275YZQP.js";
9
+ import "./chunk-7PVU2Z32.js";
10
+ import {
11
+ globPaths
12
+ } from "./chunk-AJMTRCPJ.js";
13
+ import "./chunk-JZOR5GA2.js";
14
+
15
+ // src/ingest-articles.ts
5
16
  import { readFile } from "node:fs/promises";
6
- import { basename } from "node:path";
7
- import { globPaths } from "./lib/glob-paths.js";
8
- const SCRIPT_NAME = "ingest-articles";
9
- function isUnchanged(existing, item, fileQuote, fileOgImageUrl, fileOgTitle, fileOgDescription) {
10
- return (existing.title === item.title &&
11
- existing.quote === fileQuote &&
12
- existing.ogImageUrl === fileOgImageUrl &&
13
- existing.ogTitle === fileOgTitle &&
14
- existing.ogDescription === fileOgDescription);
17
+ async function* source(root) {
18
+ for (const path of await globPaths(`${root}/json/article/*.json`)) {
19
+ const json = JSON.parse(await readFile(path, "utf-8"));
20
+ for (const item of json.body)
21
+ yield {
22
+ values: {
23
+ ...item,
24
+ categoryName: json.heading,
25
+ quote: item.quote ?? null,
26
+ ogTitle: item.ogTitle ?? null,
27
+ ogDescription: item.ogDescription ?? null,
28
+ ogImageUrl: item.ogImageUrl ?? null
29
+ }
30
+ };
31
+ }
15
32
  }
16
- function outcomeKey(outcome) {
17
- if (outcome === "inserted")
18
- return "insertedCount";
19
- if (outcome === "updated")
20
- return "updatedCount";
21
- return "skippedCount";
22
- }
23
- async function main() {
24
- const dryRun = process.argv.includes("--dry-run");
25
- const env = {
26
- DATABASE_URL: process.env.DATABASE_URL,
27
- PUSHOVER_URL: process.env.PUSHOVER_URL,
28
- PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
29
- PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
30
- USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
31
- };
32
- if (Object.values(env).some((v) => !v)) {
33
- throw new Error("Required environment variables are not set.");
34
- }
35
- const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
36
- const prisma = createPrismaClient(env.DATABASE_URL ?? "");
37
- const notificationService = createPushoverService({
38
- url: env.PUSHOVER_URL ?? "",
39
- userKey: env.PUSHOVER_USER_KEY ?? "",
40
- appToken: env.PUSHOVER_APP_TOKEN ?? "",
41
- });
42
- const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
43
- const exported = makeExportedStatus();
44
- const fileUrls = new Set();
45
- async function processExisting(existing, item) {
46
- const fileQuote = item.quote ?? null;
47
- const fileOgImageUrl = item.ogImageUrl ?? null;
48
- const fileOgTitle = item.ogTitle ?? null;
49
- const fileOgDescription = item.ogDescription ?? null;
50
- if (isUnchanged(existing, item, fileQuote, fileOgImageUrl, fileOgTitle, fileOgDescription)) {
51
- // console.log(
52
- // `ā­ļø ć‚¹ć‚­ćƒƒćƒ—ļ¼ˆå¤‰ę›“ćŖć—ļ¼‰: ${item.title} (${item.url})`,
53
- // );
54
- return "skipped";
55
- }
56
- if (dryRun) {
57
- console.log(`šŸ”„ [dry-run] ę›“ę–°äŗˆå®š: ${item.title} (${item.url})`);
58
- }
59
- else {
60
- await prisma.article.update({
61
- where: { id: existing.id },
62
- data: {
63
- title: item.title,
64
- quote: fileQuote,
65
- ogImageUrl: fileOgImageUrl,
66
- ogTitle: fileOgTitle,
67
- ogDescription: fileOgDescription,
68
- },
69
- });
70
- console.log(`šŸ”„ ꛓꖰ: ${item.title}`);
71
- }
72
- return "updated";
73
- }
74
- async function processNew(item, categoryId, existingArticlesMap) {
75
- if (dryRun) {
76
- console.log(`šŸ” [dry-run] ęŒæå…„äŗˆå®š: ${item.title} (${item.url})`);
77
- existingArticlesMap.set(item.url, {
78
- id: "",
79
- url: item.url,
80
- title: item.title,
81
- quote: item.quote ?? null,
82
- ogImageUrl: item.ogImageUrl ?? null,
83
- ogTitle: item.ogTitle ?? null,
84
- ogDescription: item.ogDescription ?? null,
85
- });
86
- return "inserted";
87
- }
88
- await prisma.article.create({
89
- data: {
90
- id: String(makeId()),
91
- title: item.title,
92
- url: item.url,
93
- quote: item.quote ?? null,
94
- ogImageUrl: item.ogImageUrl ?? null,
95
- ogTitle: item.ogTitle ?? null,
96
- ogDescription: item.ogDescription ?? null,
97
- categoryId,
98
- status: exported.status,
99
- exportedAt: exported.exportedAt,
100
- userId,
101
- createdAt: new Date(),
102
- },
103
- });
104
- existingArticlesMap.set(item.url, {
105
- id: "",
106
- url: item.url,
107
- title: item.title,
108
- quote: item.quote ?? null,
109
- ogImageUrl: item.ogImageUrl ?? null,
110
- ogTitle: item.ogTitle ?? null,
111
- ogDescription: item.ogDescription ?? null,
112
- });
113
- console.log(`āœ… ęŒæå…„: ${item.title}`);
114
- return "inserted";
115
- }
116
- async function processItem(item, categoryId, existingArticlesMap) {
117
- fileUrls.add(item.url);
118
- const existing = existingArticlesMap.get(item.url);
119
- if (existing) {
120
- return processExisting(existing, item);
121
- }
122
- return processNew(item, categoryId, existingArticlesMap);
123
- }
124
- async function resolveCategoryId(categoryName, categoryMap) {
125
- const existingId = categoryMap.get(categoryName);
126
- if (existingId) {
127
- return { categoryId: existingId, created: false };
128
- }
129
- let categoryId;
130
- if (dryRun) {
131
- console.log(`šŸ” [dry-run] ć‚«ćƒ†ć‚“ćƒŖä½œęˆäŗˆå®š: ${categoryName}`);
132
- categoryId = `dry-run-${categoryName}`;
133
- }
134
- else {
135
- const category = await prisma.category.create({
136
- data: {
137
- id: String(makeId()),
138
- name: categoryName,
139
- userId,
140
- createdAt: new Date(),
141
- },
142
- });
143
- categoryId = category.id;
144
- console.log(`šŸ“‚ ć‚«ćƒ†ć‚“ćƒŖä½œęˆ: ${categoryName}`);
145
- }
146
- categoryMap.set(categoryName, categoryId);
147
- return { categoryId, created: true };
148
- }
149
- async function ingestArticles() {
150
- const files = await globPaths(`${contentsPath}/json/article/*.json`);
151
- console.log(`šŸ“ ${files.length} 件恮JSONćƒ•ć‚”ć‚¤ćƒ«ć‚’ę¤œå‡ŗć—ć¾ć—ćŸć€‚`);
152
- const existingArticles = await prisma.article.findMany({
153
- where: { userId },
154
- select: {
155
- id: true,
156
- url: true,
157
- title: true,
158
- quote: true,
159
- ogImageUrl: true,
160
- ogTitle: true,
161
- ogDescription: true,
162
- },
163
- });
164
- const existingArticlesMap = new Map(existingArticles.map((a) => [a.url, a]));
165
- console.log(`šŸ“Š DB 恫 ${existingArticlesMap.size} ä»¶ć®ę—¢å­˜čØ˜äŗ‹ćŒć‚ć‚Šć¾ć™ć€‚`);
166
- const existingCategories = await prisma.category.findMany({
167
- where: { userId },
168
- });
169
- const categoryMap = new Map(existingCategories.map((c) => [c.name, c.id]));
170
- const counts = { insertedCount: 0, updatedCount: 0, skippedCount: 0 };
171
- let errorCount = 0;
172
- let categoryCreatedCount = 0;
173
- for (const filePath of files) {
174
- try {
175
- const content = await readFile(filePath, "utf-8");
176
- const json = JSON.parse(content);
177
- const categoryName = json.heading;
178
- const { categoryId, created } = await resolveCategoryId(categoryName, categoryMap);
179
- if (created) {
180
- categoryCreatedCount++;
181
- }
182
- for (const item of json.body) {
183
- try {
184
- const outcome = await processItem(item, categoryId, existingArticlesMap);
185
- counts[outcomeKey(outcome)]++;
186
- }
187
- catch (itemError) {
188
- console.error(`āŒ ć‚Øćƒ©ćƒ¼ļ¼ˆ${basename(filePath)} > ${item.title}):`, `url(${item.url.length}文字)=${item.url}`, `title(${item.title.length}文字)`, `quote(${(item.quote ?? "").length}文字)`, `ogImageUrl(${(item.ogImageUrl ?? "").length}文字)`, `ogTitle(${(item.ogTitle ?? "").length}文字)`, `ogDescription(${(item.ogDescription ?? "").length}文字)`, itemError);
189
- errorCount++;
190
- }
191
- }
192
- }
193
- catch (error) {
194
- console.error(`āŒ ć‚Øćƒ©ćƒ¼ļ¼ˆ${basename(filePath)}):`, error);
195
- errorCount++;
196
- }
197
- }
198
- return {
199
- insertedCount: counts.insertedCount,
200
- updatedCount: counts.updatedCount,
201
- skippedCount: counts.skippedCount,
202
- errorCount,
203
- categoryCreatedCount,
204
- };
205
- }
206
- async function purgeArticles() {
207
- const exportedArticles = await prisma.article.findMany({
208
- where: { userId, status: exported.status },
209
- select: { id: true, url: true },
210
- });
211
- const toDelete = exportedArticles.filter((a) => !fileUrls.has(a.url));
212
- if (toDelete.length === 0) {
213
- console.log("šŸ—‘ļø å‰Šé™¤åÆ¾č±”ćŖć—");
214
- return 0;
215
- }
216
- let deletedCount = 0;
217
- for (const article of toDelete) {
218
- if (dryRun) {
219
- console.log(`šŸ—‘ļø [dry-run] å‰Šé™¤äŗˆå®š: ${article.url}`);
220
- }
221
- else {
222
- await prisma.article.delete({ where: { id: article.id } });
223
- console.log(`šŸ—‘ļø 削除: ${article.url}`);
224
- }
225
- deletedCount++;
226
- }
227
- return deletedCount;
228
- }
229
- try {
230
- const { insertedCount, updatedCount, skippedCount, errorCount, categoryCreatedCount, } = await ingestArticles();
231
- const deletedCount = await purgeArticles();
232
- console.log(`\nšŸ“Š ēµęžœ: ęŒæå…„ ${insertedCount} ä»¶, ꛓꖰ ${updatedCount} ä»¶, ć‚¹ć‚­ćƒƒćƒ— ${skippedCount} ä»¶, 削除 ${deletedCount} ä»¶, ć‚Øćƒ©ćƒ¼ ${errorCount} ä»¶, ć‚«ćƒ†ć‚“ćƒŖę–°č¦ ${categoryCreatedCount} ä»¶${dryRun ? " (dry-run)" : ""}`);
233
- await notificationService.notifyInfo(`${SCRIPT_NAME} completed`, {
234
- caller: SCRIPT_NAME,
235
- });
236
- }
237
- catch (error) {
238
- console.error("āŒ ć‚Øćƒ©ćƒ¼ćŒē™ŗē”Ÿć—ć¾ć—ćŸ:", error);
239
- await notificationService.notifyError(`${SCRIPT_NAME} failed: ${String(error)}`, {
240
- caller: SCRIPT_NAME,
241
- });
242
- process.exit(1);
243
- }
244
- finally {
245
- await prisma.$disconnect();
246
- }
247
- }
248
- main().catch((error) => {
249
- console.error(error);
250
- process.exit(1);
251
- });
252
- //# sourceMappingURL=ingest-articles.js.map
33
+ await runContentImport("articles", source);
@@ -1,325 +1,68 @@
1
1
  #!/usr/bin/env node
2
- import { makeISBN } from "@s-hirano-ist/s-core/books/entities/book-entity";
3
- import { makeExportedStatus, makeId, makeUserId, } from "@s-hirano-ist/s-core/shared-kernel/entities/common-entity";
4
- import { createPrismaClient } from "@s-hirano-ist/s-database";
5
- import { createWebpThumbnail } from "@s-hirano-ist/s-image-processing/node";
6
- import { createPushoverService } from "@s-hirano-ist/s-notification";
7
- import { parseMarkdownFrontmatter } from "@s-hirano-ist/s-search/frontmatter";
8
- import { createMinioClient } from "@s-hirano-ist/s-storage";
2
+ import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
3
+ import {
4
+ makeISBN,
5
+ runContentImport
6
+ } from "./chunk-XG6LAGZO.js";
7
+ import "./chunk-KKZP7ZF3.js";
8
+ import "./chunk-F4J6JBR4.js";
9
+ import "./chunk-M275YZQP.js";
10
+ import "./chunk-7PVU2Z32.js";
11
+ import {
12
+ parseMarkdownFrontmatter
13
+ } from "./chunk-HDYQV2Y2.js";
14
+ import {
15
+ globPaths
16
+ } from "./chunk-AJMTRCPJ.js";
17
+ import "./chunk-JZOR5GA2.js";
18
+
19
+ // src/ingest-books.ts
9
20
  import { readFile } from "node:fs/promises";
10
21
  import { basename, extname } from "node:path";
11
- import { globPaths } from "./lib/glob-paths.js";
12
- const SCRIPT_NAME = "ingest-books";
13
- const ORIGINAL_BOOK_IMAGE_PATH = "books/original";
14
- const THUMBNAIL_BOOK_IMAGE_PATH = "books/thumbnail";
15
- const THUMBNAIL_SIZE = 192;
16
- function getContentType(filePath) {
17
- const ext = extname(filePath).toLowerCase();
18
- switch (ext) {
19
- case ".jpg":
20
- case ".jpeg":
21
- return "image/jpeg";
22
- case ".png":
23
- return "image/png";
24
- case ".webp":
25
- return "image/webp";
26
- default:
27
- return null;
28
- }
29
- }
30
- function arrayEquals(a, b) {
31
- if (a.length !== b.length)
32
- return false;
33
- for (const [i, element] of a.entries()) {
34
- if (element !== b[i])
35
- return false;
36
- }
37
- return true;
38
- }
39
22
  function parseBookFile(content) {
40
- const parsed = parseMarkdownFrontmatter(content);
41
- const data = parsed.data;
42
- // title: frontmatter.title ć‚’å„Ŗå…ˆć€ē„”ć‘ć‚Œć°ęœ¬ę–‡ć® H1 ć‹ć‚‰ćƒ•ć‚©ćƒ¼ćƒ«ćƒćƒƒć‚Æ
43
- let title = data.title?.trim() ?? "";
44
- if (!title) {
45
- const h1Match = /^# (.+)$/mu.exec(parsed.content);
46
- title = h1Match ? h1Match[1].trim() : "";
47
- }
48
- const body = parsed.content.trim();
49
- // oxlint-disable-next-line typescript/no-unnecessary-condition -- frontmatter is untrusted YAML; rating can be null at runtime despite the declared type
50
- if (data.rating === undefined || data.rating === null) {
51
- throw new Error("rating ćÆåæ…é ˆć§ć™ (frontmatter 恫 rating ćƒ•ć‚£ćƒ¼ćƒ«ćƒ‰ćŒć‚ć‚Šć¾ć›ć‚“)");
52
- }
53
- return {
54
- title,
55
- markdown: body || null,
56
- rating: data.rating,
57
- tags: data.tags ?? [],
58
- googleSubTitle: data.googleSubtitle ?? null,
59
- googleAuthors: data.googleAuthors ?? [],
60
- googleDescription: data.googleDescription ?? null,
61
- googleImgSrc: data.googleImgSrc ?? null,
62
- googleHref: data.googleHref ?? null,
63
- };
64
- }
65
- function bookMatchesExisting(existing, parsed, expectedImagePath) {
66
- return (existing.title === parsed.title &&
67
- existing.markdown === parsed.markdown &&
68
- existing.imagePath === expectedImagePath &&
69
- existing.rating === parsed.rating &&
70
- arrayEquals(existing.tags, parsed.tags) &&
71
- existing.googleSubTitle === parsed.googleSubTitle &&
72
- arrayEquals(existing.googleAuthors, parsed.googleAuthors) &&
73
- existing.googleDescription === parsed.googleDescription &&
74
- existing.googleImgSrc === parsed.googleImgSrc &&
75
- existing.googleHref === parsed.googleHref);
23
+ const parsed = parseMarkdownFrontmatter(content);
24
+ const data = parsed.data;
25
+ let title = data.title?.trim() ?? "";
26
+ if (!title) {
27
+ const h1Match = /^# (.+)$/mu.exec(parsed.content);
28
+ title = h1Match ? h1Match[1].trim() : "";
29
+ }
30
+ const body = parsed.content.trim();
31
+ if (data.rating === void 0 || data.rating === null) {
32
+ throw new Error(
33
+ "rating \u306F\u5FC5\u9808\u3067\u3059 (frontmatter \u306B rating \u30D5\u30A3\u30FC\u30EB\u30C9\u304C\u3042\u308A\u307E\u305B\u3093)"
34
+ );
35
+ }
36
+ return {
37
+ title,
38
+ markdown: body || null,
39
+ rating: data.rating,
40
+ tags: data.tags ?? [],
41
+ googleSubTitle: data.googleSubtitle ?? null,
42
+ googleAuthors: data.googleAuthors ?? [],
43
+ googleDescription: data.googleDescription ?? null,
44
+ googleImgSrc: data.googleImgSrc ?? null,
45
+ googleHref: data.googleHref ?? null
46
+ };
76
47
  }
77
- async function main() {
78
- const dryRun = process.argv.includes("--dry-run");
79
- const env = {
80
- DATABASE_URL: process.env.DATABASE_URL,
81
- PUSHOVER_URL: process.env.PUSHOVER_URL,
82
- PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
83
- PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
84
- USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
85
- MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME,
48
+ async function* source(root) {
49
+ const covers = /* @__PURE__ */ new Map();
50
+ for (const path of await globPaths(`${root}/image/book/*`)) {
51
+ if (!/\.(?:jpg|jpeg|png|webp)$/iu.test(path)) continue;
52
+ covers.set(makeISBN(basename(path, extname(path))), path);
53
+ }
54
+ for (const path of await globPaths(`${root}/markdown/book/*.md`)) {
55
+ const isbn = makeISBN(basename(path, ".md"));
56
+ const cover = covers.get(isbn);
57
+ yield {
58
+ values: {
59
+ ...parseBookFile(await readFile(path, "utf-8")),
60
+ isbn,
61
+ imagePath: cover ? basename(cover) : null
62
+ },
63
+ preserveImagePath: !cover,
64
+ ...cover ? { media: { path: basename(cover), bytes: await readFile(cover) } } : {}
86
65
  };
87
- if (Object.values(env).some((v) => !v)) {
88
- throw new Error("Required environment variables are not set.");
89
- }
90
- if (process.env.MINIO_USE_SSL === "true" &&
91
- (!process.env.CF_ACCESS_CLIENT_ID || !process.env.CF_ACCESS_CLIENT_SECRET)) {
92
- throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
93
- }
94
- const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
95
- const prisma = createPrismaClient(env.DATABASE_URL ?? "");
96
- const notificationService = createPushoverService({
97
- url: env.PUSHOVER_URL ?? "",
98
- userKey: env.PUSHOVER_USER_KEY ?? "",
99
- appToken: env.PUSHOVER_APP_TOKEN ?? "",
100
- });
101
- const minioClient = createMinioClient();
102
- const bucketName = env.MINIO_BUCKET_NAME ?? "";
103
- const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
104
- const exported = makeExportedStatus();
105
- const fileIsbns = new Set();
106
- async function uploadBookImage(localPath) {
107
- const fileName = basename(localPath);
108
- const buffer = await readFile(localPath);
109
- const originalKey = `${ORIGINAL_BOOK_IMAGE_PATH}/${fileName}`;
110
- try {
111
- await minioClient.statObject(bucketName, originalKey);
112
- }
113
- catch {
114
- await minioClient.putObject(bucketName, originalKey, buffer);
115
- console.log(`šŸ“¤ MinIO ć‚¢ćƒƒćƒ—ćƒ­ćƒ¼ćƒ‰ļ¼ˆoriginal): ${fileName}`);
116
- }
117
- const thumbnailBuffer = await createWebpThumbnail(buffer, {
118
- width: THUMBNAIL_SIZE,
119
- height: THUMBNAIL_SIZE,
120
- });
121
- const thumbnailKey = `${THUMBNAIL_BOOK_IMAGE_PATH}/${fileName}`;
122
- await minioClient.putObject(bucketName, thumbnailKey, Buffer.from(thumbnailBuffer));
123
- return fileName;
124
- }
125
- async function resolveNewImagePath(localImagePath, fallback) {
126
- if (!localImagePath)
127
- return fallback;
128
- if (dryRun)
129
- return basename(localImagePath);
130
- return uploadBookImage(localImagePath);
131
- }
132
- async function ingestBooks() {
133
- const files = await globPaths(`${contentsPath}/markdown/book/*.md`);
134
- console.log(`šŸ“ ${files.length} ä»¶ć®ćƒ•ć‚”ć‚¤ćƒ«ć‚’ę¤œå‡ŗć—ć¾ć—ćŸć€‚`);
135
- const imageFiles = await globPaths(`${contentsPath}/image/book/*`);
136
- const bookImageMap = new Map();
137
- for (const f of imageFiles) {
138
- if (getContentType(f) === null)
139
- continue;
140
- const rawIsbn = basename(f, extname(f));
141
- try {
142
- const isbn = makeISBN(rawIsbn);
143
- bookImageMap.set(isbn, f);
144
- }
145
- catch {
146
- console.error(`āš ļø ē„”åŠ¹ćŖē”»åƒISBN: ${basename(f)}`);
147
- }
148
- }
149
- console.log(`šŸ–¼ļø ${bookImageMap.size} ä»¶ć®ę›øē±ē”»åƒć‚’ę¤œå‡ŗć—ć¾ć—ćŸć€‚`);
150
- const existingBooks = await prisma.book.findMany({
151
- where: { userId },
152
- select: {
153
- id: true,
154
- isbn: true,
155
- title: true,
156
- markdown: true,
157
- imagePath: true,
158
- rating: true,
159
- tags: true,
160
- googleSubTitle: true,
161
- googleAuthors: true,
162
- googleDescription: true,
163
- googleImgSrc: true,
164
- googleHref: true,
165
- },
166
- });
167
- const existingBooksMap = new Map(existingBooks.map((b) => [b.isbn, b]));
168
- console.log(`šŸ“Š DB 恫 ${existingBooksMap.size} ä»¶ć®ę—¢å­˜ę›øē±ćŒć‚ć‚Šć¾ć™ć€‚`);
169
- let insertedCount = 0;
170
- let updatedCount = 0;
171
- let skippedCount = 0;
172
- let errorCount = 0;
173
- async function updateExistingBook(existing, parsed, localImagePath, isbn, title, markdown) {
174
- const newImagePath = await resolveNewImagePath(localImagePath, existing.imagePath);
175
- if (dryRun) {
176
- console.log(`šŸ”„ [dry-run] ę›“ę–°äŗˆå®š: ${isbn} (${title})`);
177
- return;
178
- }
179
- await prisma.book.update({
180
- where: { id: existing.id },
181
- data: {
182
- title,
183
- markdown,
184
- imagePath: newImagePath,
185
- rating: parsed.rating,
186
- tags: parsed.tags,
187
- googleSubTitle: parsed.googleSubTitle,
188
- googleAuthors: parsed.googleAuthors,
189
- googleDescription: parsed.googleDescription,
190
- googleImgSrc: parsed.googleImgSrc,
191
- googleHref: parsed.googleHref,
192
- },
193
- });
194
- console.log(`šŸ”„ ꛓꖰ: ${isbn} (${title})`);
195
- }
196
- async function createNewBook(parsed, localImagePath, isbn, title, markdown) {
197
- const newImagePath = await resolveNewImagePath(localImagePath, null);
198
- if (dryRun) {
199
- console.log(`šŸ” [dry-run] ęŒæå…„äŗˆå®š: ${isbn} (${title})`);
200
- return;
201
- }
202
- await prisma.book.create({
203
- data: {
204
- id: String(makeId()),
205
- isbn,
206
- title,
207
- markdown,
208
- imagePath: newImagePath,
209
- status: exported.status,
210
- exportedAt: exported.exportedAt,
211
- userId,
212
- createdAt: new Date(),
213
- rating: parsed.rating,
214
- tags: parsed.tags,
215
- googleSubTitle: parsed.googleSubTitle,
216
- googleAuthors: parsed.googleAuthors,
217
- googleDescription: parsed.googleDescription,
218
- googleImgSrc: parsed.googleImgSrc,
219
- googleHref: parsed.googleHref,
220
- },
221
- });
222
- console.log(`āœ… ęŒæå…„: ${isbn} (${title})`);
223
- }
224
- async function processBookFile(filePath) {
225
- const rawIsbn = basename(filePath, ".md");
226
- let isbn = rawIsbn;
227
- let title = "";
228
- let markdown = null;
229
- try {
230
- isbn = makeISBN(rawIsbn);
231
- fileIsbns.add(isbn);
232
- const content = await readFile(filePath, "utf-8");
233
- const parsed = parseBookFile(content);
234
- title = parsed.title;
235
- markdown = parsed.markdown;
236
- if (!title) {
237
- console.error(`āš ļø ć‚æć‚¤ćƒˆćƒ«ćŖć—: ${basename(filePath)}`);
238
- return "error";
239
- }
240
- const localImagePath = bookImageMap.get(isbn);
241
- const existing = existingBooksMap.get(isbn);
242
- if (existing) {
243
- const expectedImagePath = localImagePath
244
- ? basename(localImagePath)
245
- : existing.imagePath;
246
- if (bookMatchesExisting(existing, parsed, expectedImagePath)) {
247
- return "skipped";
248
- }
249
- await updateExistingBook(existing, parsed, localImagePath, isbn, title, markdown);
250
- return "updated";
251
- }
252
- await createNewBook(parsed, localImagePath, isbn, title, markdown);
253
- return "inserted";
254
- }
255
- catch (error) {
256
- console.error(`āŒ ć‚Øćƒ©ćƒ¼ļ¼ˆ${basename(filePath)}):`, `isbn=${isbn}`, `title(${title.length}文字)`, `markdown(${(markdown ?? "").length}文字)`, error);
257
- return "error";
258
- }
259
- }
260
- for (const filePath of files) {
261
- const result = await processBookFile(filePath);
262
- switch (result) {
263
- case "inserted":
264
- insertedCount++;
265
- break;
266
- case "updated":
267
- updatedCount++;
268
- break;
269
- case "skipped":
270
- skippedCount++;
271
- break;
272
- case "error":
273
- errorCount++;
274
- break;
275
- }
276
- }
277
- return { insertedCount, updatedCount, skippedCount, errorCount };
278
- }
279
- async function purgeBooks() {
280
- const exportedBooks = await prisma.book.findMany({
281
- where: { userId, status: exported.status },
282
- select: { id: true, isbn: true },
283
- });
284
- const toDelete = exportedBooks.filter((b) => !fileIsbns.has(b.isbn));
285
- if (toDelete.length === 0) {
286
- console.log("šŸ—‘ļø å‰Šé™¤åÆ¾č±”ćŖć—");
287
- return 0;
288
- }
289
- let deletedCount = 0;
290
- for (const book of toDelete) {
291
- if (dryRun) {
292
- console.log(`šŸ—‘ļø [dry-run] å‰Šé™¤äŗˆå®š: ${book.isbn}`);
293
- }
294
- else {
295
- await prisma.book.delete({ where: { id: book.id } });
296
- console.log(`šŸ—‘ļø 削除: ${book.isbn}`);
297
- }
298
- deletedCount++;
299
- }
300
- return deletedCount;
301
- }
302
- try {
303
- const { insertedCount, updatedCount, skippedCount, errorCount } = await ingestBooks();
304
- const deletedCount = await purgeBooks();
305
- console.log(`\nšŸ“Š ēµęžœ: ęŒæå…„ ${insertedCount} ä»¶, ꛓꖰ ${updatedCount} ä»¶, ć‚¹ć‚­ćƒƒćƒ— ${skippedCount} ä»¶, 削除 ${deletedCount} ä»¶, ć‚Øćƒ©ćƒ¼ ${errorCount} ä»¶${dryRun ? " (dry-run)" : ""}`);
306
- await notificationService.notifyInfo(`${SCRIPT_NAME} completed`, {
307
- caller: SCRIPT_NAME,
308
- });
309
- }
310
- catch (error) {
311
- console.error("āŒ ć‚Øćƒ©ćƒ¼ćŒē™ŗē”Ÿć—ć¾ć—ćŸ:", error);
312
- await notificationService.notifyError(`${SCRIPT_NAME} failed: ${String(error)}`, {
313
- caller: SCRIPT_NAME,
314
- });
315
- process.exit(1);
316
- }
317
- finally {
318
- await prisma.$disconnect();
319
- }
66
+ }
320
67
  }
321
- main().catch((error) => {
322
- console.error(error);
323
- process.exit(1);
324
- });
325
- //# sourceMappingURL=ingest-books.js.map
68
+ await runContentImport("books", source);