@s-hirano-ist/s-scripts 4.1.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +8 -0
  2. package/dist/{chunk-7XX57D7G.js → chunk-3INTMHPH.js} +1 -1
  3. package/dist/chunk-4CXH2QIX.js +445 -0
  4. package/dist/chunk-67LIMNXH.js +36 -0
  5. package/dist/{chunk-OCEV2QX3.js → chunk-BYEEIQ7O.js} +1 -1
  6. package/dist/{chunk-2PZXXOYG.js → chunk-GBQOCN2C.js} +24 -25
  7. package/dist/chunk-MJYGMGH5.js +382 -0
  8. package/dist/chunk-MVYAZJB2.js +160 -0
  9. package/dist/{chunk-DIG75T3J.js → chunk-RLAYINCA.js} +10 -13
  10. package/dist/{chunk-MNBEFEFR.js → chunk-RO5QUZFR.js} +1 -1
  11. package/dist/chunk-SFKQ2RAP.js +87 -0
  12. package/dist/{chunk-FVK44UTM.js → chunk-ZMQUO6OA.js} +1 -1
  13. package/dist/cleanup-minio-images.js +62 -91
  14. package/dist/enrich-books.js +6 -4
  15. package/dist/fetch-articles.js +6 -24
  16. package/dist/fetch-books.js +12 -35
  17. package/dist/fetch-images.js +12 -35
  18. package/dist/fetch-notes.js +6 -24
  19. package/dist/find-duplicate-json-articles.js +11 -13
  20. package/dist/fix-book-frontmatter.js +6 -4
  21. package/dist/fix-image-formats.js +7 -5
  22. package/dist/ingest-articles.js +7 -5
  23. package/dist/ingest-books.js +10 -8
  24. package/dist/ingest-images.js +7 -5
  25. package/dist/ingest-notes.js +10 -8
  26. package/dist/rag/delete-collection.js +5 -30
  27. package/dist/rag/ingest.js +5 -379
  28. package/dist/rag/search.js +5 -84
  29. package/dist/reset-articles.js +8 -23
  30. package/dist/reset-books.js +8 -23
  31. package/dist/reset-images.js +8 -23
  32. package/dist/reset-notes.js +8 -23
  33. package/dist/revert-articles.js +8 -23
  34. package/dist/revert-books.js +8 -23
  35. package/dist/revert-images.js +8 -23
  36. package/dist/revert-notes.js +8 -23
  37. package/dist/s-content-format.js +9 -11
  38. package/dist/update-json-articles.js +19 -41
  39. package/dist/update-raw-articles.js +14 -37
  40. package/dist/validate-json-articles.js +8 -5
  41. package/package.json +13 -11
  42. package/dist/chunk-7PVU2Z32.js +0 -71
  43. package/dist/chunk-F4J6JBR4.js +0 -115
@@ -7,8 +7,10 @@ import {
7
7
  resolveArticleOgImageUrl
8
8
  } from "./chunk-IYGLKY27.js";
9
9
  import {
10
- createPushoverService
11
- } from "./chunk-7PVU2Z32.js";
10
+ cliNotifications,
11
+ reportCliError,
12
+ runCli
13
+ } from "./chunk-4CXH2QIX.js";
12
14
  import "./chunk-JZOR5GA2.js";
13
15
 
14
16
  // src/update-json-articles.ts
@@ -43,6 +45,7 @@ async function getOgTags(url) {
43
45
  const ogDescription = ogTags["og:description"];
44
46
  return { ogImageUrl, ogTitle, ogDescription };
45
47
  } catch (error) {
48
+ reportCliError(error, "update-json-articles");
46
49
  console.error(`Error fetching OG tags for ${url}:`, error);
47
50
  return {};
48
51
  }
@@ -73,49 +76,24 @@ async function processArticleFile(filePath) {
73
76
  );
74
77
  console.log(`Completed processing ${filePath}`);
75
78
  } catch (error) {
79
+ reportCliError(error, "update-json-articles");
76
80
  console.error(`Error processing ${filePath}:`, error);
77
81
  }
78
82
  }
79
83
  async function main() {
80
- const env = {
81
- PUSHOVER_URL: process.env.PUSHOVER_URL,
82
- PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
83
- PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN
84
- };
85
- if (Object.values(env).some((v) => !v)) {
86
- throw new Error("Required environment variables are not set.");
87
- }
88
- const notificationService = createPushoverService({
89
- url: env.PUSHOVER_URL ?? "",
90
- userKey: env.PUSHOVER_USER_KEY ?? "",
91
- appToken: env.PUSHOVER_APP_TOKEN ?? ""
92
- });
84
+ const notificationService = cliNotifications;
93
85
  const articleDir = join(process.cwd(), "json", "article");
94
- try {
95
- const files = readdirSync(articleDir).filter(
96
- (file) => file.endsWith(".json")
97
- );
98
- console.log(`Found ${files.length} JSON files to process`);
99
- for (const file of files) {
100
- const filePath = join(articleDir, file);
101
- await processArticleFile(filePath);
102
- }
103
- console.log("All files processed successfully!");
104
- await notificationService.notifyInfo("update-json-articles completed", {
105
- caller: "update-json-articles"
106
- });
107
- } catch (error) {
108
- console.error("Error in main process:", error);
109
- await notificationService.notifyError(
110
- `update-json-articles failed: ${String(error)}`,
111
- {
112
- caller: "update-json-articles"
113
- }
114
- );
115
- process.exit(1);
86
+ const files = readdirSync(articleDir).filter(
87
+ (file) => file.endsWith(".json")
88
+ );
89
+ console.log(`Found ${files.length} JSON files to process`);
90
+ for (const file of files) {
91
+ const filePath = join(articleDir, file);
92
+ await processArticleFile(filePath);
116
93
  }
94
+ console.log("All files processed successfully!");
95
+ await notificationService.notifyInfo("update-json-articles completed", {
96
+ caller: "update-json-articles"
97
+ });
117
98
  }
118
- main().catch((error) => {
119
- console.error(error);
120
- process.exit(1);
121
- });
99
+ await runCli("update-json-articles", main);
@@ -4,8 +4,10 @@ import {
4
4
  decodeHtml
5
5
  } from "./chunk-UXGLXHQ6.js";
6
6
  import {
7
- createPushoverService
8
- } from "./chunk-7PVU2Z32.js";
7
+ cliNotifications,
8
+ reportCliError,
9
+ runCli
10
+ } from "./chunk-4CXH2QIX.js";
9
11
  import "./chunk-JZOR5GA2.js";
10
12
 
11
13
  // src/update-raw-articles.ts
@@ -73,6 +75,7 @@ async function fetchWebsiteMarkdown(url) {
73
75
  const markdown = turndownService.turndown(html);
74
76
  return markdown.trim();
75
77
  } catch (error) {
78
+ reportCliError(error, "update-raw-articles");
76
79
  console.error(`Failed to fetch ${url}:`, error);
77
80
  return "";
78
81
  }
@@ -118,41 +121,15 @@ ${websiteText}
118
121
  }
119
122
  }
120
123
  async function main() {
121
- const env = {
122
- PUSHOVER_URL: process.env.PUSHOVER_URL,
123
- PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
124
- PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN
125
- };
126
- if (Object.values(env).some((v) => !v)) {
127
- throw new Error("Required environment variables are not set.");
124
+ const notificationService = cliNotifications;
125
+ const jsonFiles = await readdir(JSON_DIR);
126
+ const jsonFilePaths = jsonFiles.filter((file) => file.endsWith(".json")).map((file) => path.join(JSON_DIR, file));
127
+ for (const jsonFile of jsonFilePaths) {
128
+ console.log(`Processing: ${jsonFile}`);
129
+ await jsonToMarkdown(jsonFile);
128
130
  }
129
- const notificationService = createPushoverService({
130
- url: env.PUSHOVER_URL ?? "",
131
- userKey: env.PUSHOVER_USER_KEY ?? "",
132
- appToken: env.PUSHOVER_APP_TOKEN ?? ""
131
+ await notificationService.notifyInfo("update-raw-articles completed", {
132
+ caller: "update-raw-articles"
133
133
  });
134
- try {
135
- const jsonFiles = await readdir(JSON_DIR);
136
- const jsonFilePaths = jsonFiles.filter((file) => file.endsWith(".json")).map((file) => path.join(JSON_DIR, file));
137
- for (const jsonFile of jsonFilePaths) {
138
- console.log(`Processing: ${jsonFile}`);
139
- await jsonToMarkdown(jsonFile);
140
- }
141
- await notificationService.notifyInfo("update-raw-articles completed", {
142
- caller: "update-raw-articles"
143
- });
144
- } catch (error) {
145
- console.error("Error in main process:", error);
146
- await notificationService.notifyError(
147
- `update-raw-articles failed: ${String(error)}`,
148
- {
149
- caller: "update-raw-articles"
150
- }
151
- );
152
- process.exit(1);
153
- }
154
134
  }
155
- main().catch((error) => {
156
- console.error(error);
157
- process.exit(1);
158
- });
135
+ await runCli("update-raw-articles", main);
@@ -1,5 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
3
+ import {
4
+ reportCliError,
5
+ runCli
6
+ } from "./chunk-4CXH2QIX.js";
7
+ import "./chunk-JZOR5GA2.js";
3
8
 
4
9
  // src/validate-json-articles.ts
5
10
  import fs from "node:fs";
@@ -117,6 +122,7 @@ async function main() {
117
122
  );
118
123
  }
119
124
  } catch (error) {
125
+ reportCliError(error, "validate-json-articles");
120
126
  if (error instanceof SyntaxError) {
121
127
  allIssues.push({
122
128
  file: fileName,
@@ -137,10 +143,7 @@ async function main() {
137
143
  displayResults(allIssues);
138
144
  const errorCount = allIssues.filter((i) => i.level === "error").length;
139
145
  if (errorCount > 0) {
140
- process.exit(1);
146
+ process.exitCode = 1;
141
147
  }
142
148
  }
143
- main().catch((error) => {
144
- console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
145
- process.exit(1);
146
- });
149
+ await runCli("validate-json-articles", main);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@s-hirano-ist/s-scripts",
3
- "version": "4.1.0",
3
+ "version": "4.3.0",
4
4
  "description": "CLI scripts for s-private data operations",
5
5
  "engines": {
6
6
  "node": "26.x"
@@ -49,19 +49,21 @@
49
49
  },
50
50
  "license": "AGPL-3.0",
51
51
  "dependencies": {
52
+ "@aws-sdk/client-s3": "3.1166.0",
52
53
  "@googleapis/books": "13.0.1",
53
- "@prisma/adapter-pg": "7.10.0",
54
- "@prisma/client-runtime-utils": "7.10.0",
55
- "@qdrant/js-client-rest": "1.19.0",
54
+ "@prisma/adapter-pg": "7.10.1",
55
+ "@prisma/client-runtime-utils": "7.10.1",
56
+ "@qdrant/js-client-rest": "1.20.0",
57
+ "@sentry/node": "11.12.0",
56
58
  "cheerio": "1.2.0",
57
- "js-yaml": "5.4.2",
58
- "minio": "8.0.7",
59
- "oxfmt": "0.72.0",
59
+ "js-yaml": "5.5.0",
60
+ "oxfmt": "0.78.0",
61
+ "pino": "10.4.0",
60
62
  "sharp": "0.35.5",
61
- "turndown": "7.2.4",
62
- "uuid": "14.0.2",
63
- "web-push": "3.6.7",
64
- "zod": "4.6.5"
63
+ "turndown": "7.2.5",
64
+ "uuid": "14.0.3",
65
+ "web-push": "4.0.0",
66
+ "zod": "4.6.7"
65
67
  },
66
68
  "publishConfig": {
67
69
  "access": "public"
@@ -1,71 +0,0 @@
1
- import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
2
-
3
- // ../notification/dist/errors.js
4
- var NotificationError = class extends Error {
5
- constructor(message = "notificationSend") {
6
- super(message);
7
- this.name = "NotificationError";
8
- }
9
- };
10
-
11
- // ../notification/dist/pushover.js
12
- var PushoverService = class {
13
- config;
14
- constructor(config) {
15
- this.config = config;
16
- }
17
- /**
18
- * Sends a message to the Pushover API.
19
- *
20
- * @param message - The message to send
21
- * @param priority - The priority level (-1 = low, 0 = normal, 1 = high)
22
- *
23
- * @internal
24
- */
25
- async sendMessage(message, priority = 0) {
26
- try {
27
- const body = new URLSearchParams({
28
- token: this.config.appToken,
29
- user: this.config.userKey,
30
- message,
31
- priority: String(priority)
32
- });
33
- const result = await fetch(this.config.url, {
34
- method: "POST",
35
- headers: { "Content-Type": "application/x-www-form-urlencoded" },
36
- body
37
- });
38
- if (!result.ok) {
39
- throw new NotificationError();
40
- }
41
- } catch (error) {
42
- if (error instanceof NotificationError) {
43
- console.error(`[NotificationService] Failed to send notification: ${error.message}`);
44
- } else {
45
- console.error("[NotificationService] Unknown error occurred while sending notification:", error);
46
- }
47
- }
48
- }
49
- async notifyError(message, context) {
50
- const enrichedMessage = `[ERROR] ${context.caller}: ${message}`;
51
- await this.sendMessage(enrichedMessage, 1);
52
- }
53
- async notifyWarning(message, context) {
54
- const enrichedMessage = `[WARN] ${context.caller}: ${message}`;
55
- await this.sendMessage(enrichedMessage, 0);
56
- }
57
- async notifyInfo(message, context) {
58
- const enrichedMessage = `[INFO] ${context.caller}: ${message}`;
59
- await this.sendMessage(enrichedMessage, -1);
60
- }
61
- };
62
- function createPushoverService(config) {
63
- return new PushoverService(config);
64
- }
65
-
66
- // ../notification/dist/web-push.js
67
- import webPush from "web-push";
68
-
69
- export {
70
- createPushoverService
71
- };
@@ -1,115 +0,0 @@
1
- import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
2
-
3
- // ../storage/dist/src/client.js
4
- import { Client } from "minio";
5
-
6
- // ../storage/dist/src/cf-access-transport.js
7
- import * as https from "node:https";
8
- function createCfAccessTransport(config) {
9
- if (!config?.clientId || !config.clientSecret) {
10
- return void 0;
11
- }
12
- return {
13
- request(options, ...rest) {
14
- if (typeof options === "object" && !(options instanceof URL)) {
15
- const headers = options.headers ?? {};
16
- headers["CF-Access-Client-Id"] = config.clientId;
17
- headers["CF-Access-Client-Secret"] = config.clientSecret;
18
- options.headers = headers;
19
- }
20
- const request2 = https.request;
21
- return request2(options, ...rest);
22
- }
23
- };
24
- }
25
-
26
- // ../storage/dist/src/client.js
27
- function createMinioClient(config, cfAccess) {
28
- const resolved = config ?? {
29
- endPoint: process.env.MINIO_HOST ?? "",
30
- port: Number(process.env.MINIO_PORT),
31
- useSSL: process.env.MINIO_USE_SSL === void 0 ? !["localhost", "127.0.0.1"].includes(process.env.MINIO_HOST ?? "") : process.env.MINIO_USE_SSL === "true",
32
- accessKey: process.env.MINIO_ACCESS_KEY ?? "",
33
- secretKey: process.env.MINIO_SECRET_KEY ?? ""
34
- };
35
- const resolvedCfAccess = cfAccess ?? {
36
- clientId: process.env.CF_ACCESS_CLIENT_ID ?? "",
37
- clientSecret: process.env.CF_ACCESS_CLIENT_SECRET ?? ""
38
- };
39
- const transport = resolved.useSSL ? createCfAccessTransport(resolvedCfAccess) : void 0;
40
- if (resolved.useSSL && !transport) {
41
- throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when useSSL is true.");
42
- }
43
- return new Client({
44
- endPoint: resolved.endPoint,
45
- port: resolved.port,
46
- useSSL: resolved.useSSL,
47
- accessKey: resolved.accessKey,
48
- secretKey: resolved.secretKey,
49
- ...transport && { transport }
50
- });
51
- }
52
-
53
- // ../storage/dist/index.js
54
- import { S3Error } from "minio";
55
-
56
- // ../storage/dist/src/errors.js
57
- var StorageOperationError = class extends Error {
58
- context;
59
- constructor(context, cause) {
60
- super(`Storage ${context.operation} failed for ${context.objectKey}`, {
61
- cause
62
- });
63
- this.name = "StorageOperationError";
64
- this.context = context;
65
- }
66
- };
67
-
68
- // ../storage/dist/src/storage-service.js
69
- function createStorageService(client, bucketName, pathConfig) {
70
- function buildObjectKey(path, isThumbnail) {
71
- return `${isThumbnail ? pathConfig.thumbnailPrefix : pathConfig.originalPrefix}/${path}`;
72
- }
73
- function wrapStorageError(operation, objectKey, isThumbnail, error) {
74
- throw new StorageOperationError({ operation, objectKey, bucketName, isThumbnail }, error);
75
- }
76
- return {
77
- async uploadImage(path, bytes, isThumbnail) {
78
- const objKey = buildObjectKey(path, isThumbnail);
79
- try {
80
- await client.putObject(bucketName, objKey, Buffer.from(bytes));
81
- } catch (error) {
82
- wrapStorageError("uploadImage", objKey, isThumbnail, error);
83
- }
84
- },
85
- async getImage(path, isThumbnail) {
86
- const objKey = buildObjectKey(path, isThumbnail);
87
- try {
88
- return await client.getObject(bucketName, objKey);
89
- } catch (error) {
90
- return wrapStorageError("getImage", objKey, isThumbnail, error);
91
- }
92
- },
93
- async getImageOrThrow(path, isThumbnail) {
94
- const objKey = buildObjectKey(path, isThumbnail);
95
- try {
96
- await client.statObject(bucketName, objKey);
97
- } catch (error) {
98
- wrapStorageError("getImageOrThrow", objKey, isThumbnail, error);
99
- }
100
- },
101
- async deleteImage(path, isThumbnail) {
102
- const objKey = buildObjectKey(path, isThumbnail);
103
- try {
104
- await client.removeObject(bucketName, objKey);
105
- } catch (error) {
106
- wrapStorageError("deleteImage", objKey, isThumbnail, error);
107
- }
108
- }
109
- };
110
- }
111
-
112
- export {
113
- createMinioClient,
114
- createStorageService
115
- };