@s-hirano-ist/s-scripts 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cleanup-minio-images.js +8 -12
- package/dist/cleanup-minio-images.js.map +1 -1
- package/dist/fetch-books.js +8 -12
- package/dist/fetch-books.js.map +1 -1
- package/dist/fetch-images.js +8 -12
- package/dist/fetch-images.js.map +1 -1
- package/dist/ingest-images.js +8 -12
- package/dist/ingest-images.js.map +1 -1
- package/dist/rag/ingest-config.d.ts +3 -2
- package/dist/rag/ingest-config.d.ts.map +1 -1
- package/dist/rag/ingest-config.js +3 -2
- package/dist/rag/ingest-config.js.map +1 -1
- package/dist/rag/ingest.js +28 -20
- package/dist/rag/ingest.js.map +1 -1
- package/dist/rag/search.js +14 -2
- package/dist/rag/search.js.map +1 -1
- package/dist/update-json-articles.js +44 -1
- package/dist/update-json-articles.js.map +1 -1
- package/package.json +6 -6
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { basename, extname } from "node:path";
|
|
3
3
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
4
|
+
import { createMinioClient } from "@s-hirano-ist/s-storage";
|
|
4
5
|
import { glob } from "glob";
|
|
5
|
-
import * as Minio from "minio";
|
|
6
6
|
const SCRIPT_NAME = "cleanup-minio-images";
|
|
7
7
|
const ORIGINAL_IMAGE_PATH = "images/original";
|
|
8
8
|
const THUMBNAIL_IMAGE_PATH = "images/thumbnail";
|
|
@@ -28,28 +28,24 @@ async function main() {
|
|
|
28
28
|
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
29
29
|
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
30
30
|
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
31
|
-
MINIO_HOST: process.env.MINIO_HOST,
|
|
32
|
-
MINIO_PORT: process.env.MINIO_PORT,
|
|
33
31
|
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME,
|
|
34
|
-
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
|
35
|
-
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
|
36
32
|
};
|
|
37
33
|
if (Object.values(env).some((v) => !v)) {
|
|
38
34
|
throw new Error("Required environment variables are not set.");
|
|
39
35
|
}
|
|
36
|
+
if (process.env.MINIO_USE_SSL === "true") {
|
|
37
|
+
if (!process.env.CF_ACCESS_CLIENT_ID ||
|
|
38
|
+
!process.env.CF_ACCESS_CLIENT_SECRET) {
|
|
39
|
+
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
40
42
|
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
41
43
|
const notificationService = createPushoverService({
|
|
42
44
|
url: env.PUSHOVER_URL ?? "",
|
|
43
45
|
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
44
46
|
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
45
47
|
});
|
|
46
|
-
const minioClient =
|
|
47
|
-
endPoint: env.MINIO_HOST ?? "",
|
|
48
|
-
port: Number(env.MINIO_PORT),
|
|
49
|
-
useSSL: true,
|
|
50
|
-
accessKey: env.MINIO_ACCESS_KEY ?? "",
|
|
51
|
-
secretKey: env.MINIO_SECRET_KEY ?? "",
|
|
52
|
-
});
|
|
48
|
+
const minioClient = createMinioClient();
|
|
53
49
|
const bucketName = env.MINIO_BUCKET_NAME ?? "";
|
|
54
50
|
try {
|
|
55
51
|
// 1. ローカルファイル一覧取得
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup-minio-images.js","sourceRoot":"","sources":["../src/cleanup-minio-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"cleanup-minio-images.js","sourceRoot":"","sources":["../src/cleanup-minio-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAoB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,MAAM,WAAW,GAAG,sBAAsB,CAAC;AAE3C,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AAC9C,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAEhD,SAAS,WAAW,CAAC,QAAgB;IACpC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,WAAW,CACnB,MAAmB,EACnB,MAAc,EACd,MAAc;IAEd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC1D,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,IAAI,GAAG,CAAC,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;KACvC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC1C,IACC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAChC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,CAAC;YACF,MAAM,IAAI,KAAK,CACd,0FAA0F,CAC1F,CAAC;QACH,CAAC;IACF,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAElE,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAC;IAExC,MAAM,UAAU,GAAG,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAE/C,IAAI,CAAC;QACJ,kBAAkB;QAClB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,GAAG,YAAY,eAAe,CAAC,CAAC;QAC9D,MAAM,cAAc,GAAG,IAAI,GAAG,CAC7B,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACtD,CAAC;QACF,OAAO,CAAC,GAAG,CACV,YAAY,cAAc,CAAC,IAAI,mBAAmB,CAClD,CAAC;QAEF,qBAAqB;QACrB,MAAM,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvD,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,mBAAmB,GAAG,CAAC;YAC/D,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,oBAAoB,GAAG,CAAC;SAChE,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CACV,sBAAsB,YAAY,CAAC,MAAM,iBAAiB,aAAa,CAAC,MAAM,IAAI,CAClF,CAAC;QAEF,UAAU;QACV,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAC5C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAC3C,CAAC;QACF,MAAM,kBAAkB,GAAG,aAAa,CAAC,MAAM,CAC9C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAC3C,CAAC;QAEF,MAAM,aAAa,GAAG,iBAAiB,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;QAC3E,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAC1B,MAAM,mBAAmB,CAAC,UAAU,CACnC,GAAG,WAAW,+BAA+B,EAC7C,EAAE,MAAM,EAAE,WAAW,EAAE,CACvB,CAAC;YACF,OAAO;QACR,CAAC;QAED,OAAO,CAAC,GAAG,CACV,2BAA2B,iBAAiB,CAAC,MAAM,iBAAiB,kBAAkB,CAAC,MAAM,IAAI,CACjG,CAAC;QAEF,UAAU;QACV,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,iBAAiB,EAAE,GAAG,kBAAkB,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC;gBACJ,IAAI,MAAM,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACP,MAAM,WAAW,CAAC,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;oBAChD,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAChC,CAAC;gBACD,YAAY,EAAE,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;gBACzC,UAAU,EAAE,CAAC;YACd,CAAC;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CACV,eAAe,YAAY,WAAW,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CACjF,CAAC;QACF,MAAM,mBAAmB,CAAC,UAAU,CAAC,GAAG,WAAW,YAAY,EAAE;YAChE,MAAM,EAAE,WAAW;SACnB,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,mBAAmB,CAAC,WAAW,CAAC,GAAG,WAAW,YAAY,KAAK,EAAE,EAAE;YACxE,MAAM,EAAE,WAAW;SACnB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/fetch-books.js
CHANGED
|
@@ -3,7 +3,7 @@ import { access, mkdir, writeFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { makeUnexportedStatus, makeUserId, } from "@s-hirano-ist/s-core/shared-kernel/entities/common-entity";
|
|
5
5
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
6
|
-
import
|
|
6
|
+
import { createMinioClient } from "@s-hirano-ist/s-storage";
|
|
7
7
|
const OUTPUT_DIR = "markdown/book/";
|
|
8
8
|
async function main() {
|
|
9
9
|
const env = {
|
|
@@ -12,15 +12,17 @@ async function main() {
|
|
|
12
12
|
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
13
13
|
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
14
14
|
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
15
|
-
MINIO_HOST: process.env.MINIO_HOST,
|
|
16
|
-
MINIO_PORT: process.env.MINIO_PORT,
|
|
17
15
|
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME,
|
|
18
|
-
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
|
19
|
-
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
|
20
16
|
};
|
|
21
17
|
if (Object.values(env).some((v) => !v)) {
|
|
22
18
|
throw new Error("Required environment variables are not set.");
|
|
23
19
|
}
|
|
20
|
+
if (process.env.MINIO_USE_SSL === "true") {
|
|
21
|
+
if (!process.env.CF_ACCESS_CLIENT_ID ||
|
|
22
|
+
!process.env.CF_ACCESS_CLIENT_SECRET) {
|
|
23
|
+
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
24
26
|
// Dynamic import for Prisma ESM compatibility
|
|
25
27
|
const { PrismaClient } = await import("@s-hirano-ist/s-database/generated");
|
|
26
28
|
const prisma = new PrismaClient({ accelerateUrl: env.DATABASE_URL ?? "" });
|
|
@@ -29,13 +31,7 @@ async function main() {
|
|
|
29
31
|
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
30
32
|
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
31
33
|
});
|
|
32
|
-
const minioClient =
|
|
33
|
-
endPoint: env.MINIO_HOST ?? "",
|
|
34
|
-
port: Number(env.MINIO_PORT),
|
|
35
|
-
useSSL: true,
|
|
36
|
-
accessKey: env.MINIO_ACCESS_KEY ?? "",
|
|
37
|
-
secretKey: env.MINIO_SECRET_KEY ?? "",
|
|
38
|
-
});
|
|
34
|
+
const minioClient = createMinioClient();
|
|
39
35
|
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
40
36
|
const UNEXPORTED = makeUnexportedStatus();
|
|
41
37
|
async function exportData(data) {
|
package/dist/fetch-books.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-books.js","sourceRoot":"","sources":["../src/fetch-books.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACN,oBAAoB,EACpB,UAAU,GAGV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,
|
|
1
|
+
{"version":3,"file":"fetch-books.js","sourceRoot":"","sources":["../src/fetch-books.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACN,oBAAoB,EACpB,UAAU,GAGV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAS5D,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAEpC,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;KACvC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC1C,IACC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAChC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,CAAC;YACF,MAAM,IAAI,KAAK,CACd,0FAA0F,CAC1F,CAAC;QACH,CAAC;IACF,CAAC;IAED,8CAA8C;IAC9C,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC,CAAC;IAE3E,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAC;IAExC,MAAM,MAAM,GAAW,UAAU,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,UAAU,GAAW,oBAAoB,EAAE,CAAC;IAElD,KAAK,UAAU,UAAU,CAAC,IAAY;QACrC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE7C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,GAAG,UAAU,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC;YAEhD,qBAAqB;YACrB,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACvB,YAAY,EAAE,CAAC;gBACf,SAAS;YACV,CAAC;YAAC,MAAM,CAAC;gBACR,oBAAoB;YACrB,CAAC;YAED,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;YAC/C,aAAa,EAAE,CAAC;QACjB,CAAC;QAED,OAAO,CAAC,GAAG,CACV,gBAAgB,aAAa,WAAW,YAAY,QAAQ,CAC5D,CAAC;IACH,CAAC;IAED,KAAK,UAAU,kBAAkB,CAAC,KAAa;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CACV,MAAM,eAAe,CAAC,MAAM,oBAAoB,CAChD,CAAC;QAEF,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAC3C,KAAK,EAAE,IAAgD,EAAE,EAAE;YAC1D,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;YACjC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAEhD,qBAAqB;YACrB,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACvB,YAAY,EAAE,CAAC;gBACf,OAAO;YACR,CAAC;YAAC,MAAM,CAAC;gBACR,sBAAsB;YACvB,CAAC;YAED,MAAM,WAAW,CAAC,UAAU,CAC3B,GAAG,CAAC,iBAAiB,IAAI,EAAE,EAC3B,kBAAkB,SAAS,EAAE,EAC7B,QAAQ,CACR,CAAC;YACF,eAAe,EAAE,CAAC;QACnB,CAAC,CACD,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CACV,YAAY,eAAe,aAAa,YAAY,QAAQ,CAC5D,CAAC;IACH,CAAC;IAED,KAAK,UAAU,UAAU;QACxB,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;YACxC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE;SACrC,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,MAAM,gBAAgB,CAAC,CAAC;QAEhD,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,UAAU,EAAE,CAAC;QACnB,MAAM,mBAAmB,CAAC,UAAU,CAAC,uBAAuB,EAAE;YAC7D,MAAM,EAAE,aAAa;SACrB,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,mBAAmB,CAAC,WAAW,CAAC,uBAAuB,KAAK,EAAE,EAAE;YACrE,MAAM,EAAE,aAAa;SACrB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/fetch-images.js
CHANGED
|
@@ -3,7 +3,7 @@ import { mkdir } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { makeUnexportedStatus, makeUserId, } from "@s-hirano-ist/s-core/shared-kernel/entities/common-entity";
|
|
5
5
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
6
|
-
import
|
|
6
|
+
import { createMinioClient } from "@s-hirano-ist/s-storage";
|
|
7
7
|
async function main() {
|
|
8
8
|
const env = {
|
|
9
9
|
DATABASE_URL: process.env.DATABASE_URL,
|
|
@@ -11,15 +11,17 @@ async function main() {
|
|
|
11
11
|
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
12
12
|
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
13
13
|
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
14
|
-
MINIO_HOST: process.env.MINIO_HOST,
|
|
15
|
-
MINIO_PORT: process.env.MINIO_PORT,
|
|
16
14
|
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME,
|
|
17
|
-
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
|
18
|
-
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
|
19
15
|
};
|
|
20
16
|
if (Object.values(env).some((v) => !v)) {
|
|
21
17
|
throw new Error("Required environment variables are not set.");
|
|
22
18
|
}
|
|
19
|
+
if (process.env.MINIO_USE_SSL === "true") {
|
|
20
|
+
if (!process.env.CF_ACCESS_CLIENT_ID ||
|
|
21
|
+
!process.env.CF_ACCESS_CLIENT_SECRET) {
|
|
22
|
+
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
|
|
23
|
+
}
|
|
24
|
+
}
|
|
23
25
|
// Dynamic import for Prisma ESM compatibility
|
|
24
26
|
const { PrismaClient } = await import("@s-hirano-ist/s-database/generated");
|
|
25
27
|
const prisma = new PrismaClient({ accelerateUrl: env.DATABASE_URL ?? "" });
|
|
@@ -28,13 +30,7 @@ async function main() {
|
|
|
28
30
|
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
29
31
|
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
30
32
|
});
|
|
31
|
-
const minioClient =
|
|
32
|
-
endPoint: env.MINIO_HOST ?? "",
|
|
33
|
-
port: Number(env.MINIO_PORT),
|
|
34
|
-
useSSL: true,
|
|
35
|
-
accessKey: env.MINIO_ACCESS_KEY ?? "",
|
|
36
|
-
secretKey: env.MINIO_SECRET_KEY ?? "",
|
|
37
|
-
});
|
|
33
|
+
const minioClient = createMinioClient();
|
|
38
34
|
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
39
35
|
const UNEXPORTED = makeUnexportedStatus();
|
|
40
36
|
async function fetchImages() {
|
package/dist/fetch-images.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-images.js","sourceRoot":"","sources":["../src/fetch-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACN,oBAAoB,EACpB,UAAU,GAGV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,
|
|
1
|
+
{"version":3,"file":"fetch-images.js","sourceRoot":"","sources":["../src/fetch-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACN,oBAAoB,EACpB,UAAU,GAGV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;KACvC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC1C,IACC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAChC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,CAAC;YACF,MAAM,IAAI,KAAK,CACd,0FAA0F,CAC1F,CAAC;QACH,CAAC;IACF,CAAC;IAED,8CAA8C;IAC9C,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC,CAAC;IAE3E,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAC;IAExC,MAAM,MAAM,GAAW,UAAU,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,UAAU,GAAW,oBAAoB,EAAE,CAAC;IAElD,KAAK,UAAU,WAAW;QACzB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC1C,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE;SACrC,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,MAAM,gBAAgB,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAuB,EAAE,EAAE;YACrE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAEjD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,MAAM,WAAW,CAAC,UAAU,CAC3B,GAAG,CAAC,iBAAiB,IAAI,EAAE,EAC3B,mBAAmB,SAAS,EAAE,EAC9B,QAAQ,CACR,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,WAAW,EAAE,CAAC;QACpB,MAAM,mBAAmB,CAAC,UAAU,CAAC,wBAAwB,EAAE;YAC9D,MAAM,EAAE,cAAc;SACtB,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,mBAAmB,CAAC,WAAW,CAAC,wBAAwB,KAAK,EAAE,EAAE;YACtE,MAAM,EAAE,cAAc;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/ingest-images.js
CHANGED
|
@@ -3,8 +3,8 @@ import { readFile, stat } from "node:fs/promises";
|
|
|
3
3
|
import { basename, extname } from "node:path";
|
|
4
4
|
import { makeExportedStatus, makeId, makeUserId, } from "@s-hirano-ist/s-core/shared-kernel/entities/common-entity";
|
|
5
5
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
6
|
+
import { createMinioClient } from "@s-hirano-ist/s-storage";
|
|
6
7
|
import { glob } from "glob";
|
|
7
|
-
import * as Minio from "minio";
|
|
8
8
|
import sharp from "sharp";
|
|
9
9
|
const SCRIPT_NAME = "ingest-images";
|
|
10
10
|
const ORIGINAL_IMAGE_PATH = "images/original";
|
|
@@ -34,15 +34,17 @@ async function main() {
|
|
|
34
34
|
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
35
35
|
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
36
36
|
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
37
|
-
MINIO_HOST: process.env.MINIO_HOST,
|
|
38
|
-
MINIO_PORT: process.env.MINIO_PORT,
|
|
39
37
|
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME,
|
|
40
|
-
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
|
41
|
-
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
|
42
38
|
};
|
|
43
39
|
if (Object.values(env).some((v) => !v)) {
|
|
44
40
|
throw new Error("Required environment variables are not set.");
|
|
45
41
|
}
|
|
42
|
+
if (process.env.MINIO_USE_SSL === "true") {
|
|
43
|
+
if (!process.env.CF_ACCESS_CLIENT_ID ||
|
|
44
|
+
!process.env.CF_ACCESS_CLIENT_SECRET) {
|
|
45
|
+
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true.");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
46
48
|
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
47
49
|
// Dynamic import for Prisma ESM compatibility
|
|
48
50
|
const { PrismaClient } = await import("@s-hirano-ist/s-database/generated");
|
|
@@ -52,13 +54,7 @@ async function main() {
|
|
|
52
54
|
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
53
55
|
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
54
56
|
});
|
|
55
|
-
const minioClient =
|
|
56
|
-
endPoint: env.MINIO_HOST ?? "",
|
|
57
|
-
port: Number(env.MINIO_PORT),
|
|
58
|
-
useSSL: true,
|
|
59
|
-
accessKey: env.MINIO_ACCESS_KEY ?? "",
|
|
60
|
-
secretKey: env.MINIO_SECRET_KEY ?? "",
|
|
61
|
-
});
|
|
57
|
+
const minioClient = createMinioClient();
|
|
62
58
|
const bucketName = env.MINIO_BUCKET_NAME ?? "";
|
|
63
59
|
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
64
60
|
const exported = makeExportedStatus();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-images.js","sourceRoot":"","sources":["../src/ingest-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EACN,kBAAkB,EAClB,MAAM,EACN,UAAU,GAEV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ingest-images.js","sourceRoot":"","sources":["../src/ingest-images.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EACN,kBAAkB,EAClB,MAAM,EACN,UAAU,GAEV,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,WAAW,GAAG,eAAe,CAAC;AAEpC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AAC9C,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAChD,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,SAAS,cAAc,CAAC,QAAgB;IACvC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACX,OAAO,YAAY,CAAC;QACrB,KAAK,MAAM;YACV,OAAO,WAAW,CAAC;QACpB,KAAK,MAAM;YACV,OAAO,WAAW,CAAC;QACpB,KAAK,OAAO;YACX,OAAO,YAAY,CAAC;QACrB;YACC,OAAO,IAAI,CAAC;IACd,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAClD,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;KACvC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC1C,IACC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAChC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,CAAC;YACF,MAAM,IAAI,KAAK,CACd,0FAA0F,CAC1F,CAAC;QACH,CAAC;IACF,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAElE,8CAA8C;IAC9C,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC,CAAC;IAE3E,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAC;IAExC,MAAM,UAAU,GAAG,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAE/C,MAAM,MAAM,GAAW,UAAU,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,KAAK,UAAU,YAAY;QAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,YAAY,eAAe,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;YAClD,KAAK,EAAE,EAAE,MAAM,EAAE;YACjB,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAChC,cAAc,CAAC,GAAG,CAAC,CAAC,CAA+B,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CACpE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,WAAW,iBAAiB,CAAC,IAAI,eAAe,CAAC,CAAC;QAE9D,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACrC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEzB,IAAI,CAAC;gBACJ,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBACtC,YAAY,EAAE,CAAC;oBACf,SAAS;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAChD,MAAM,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;oBAClB,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAC;oBAC7C,YAAY,EAAE,CAAC;oBACf,SAAS;gBACV,CAAC;gBAED,IAAI,MAAM,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;oBAC/C,aAAa,EAAE,CAAC;oBAChB,SAAS;gBACV,CAAC;gBAED,yCAAyC;gBACzC,MAAM,WAAW,GAAG,GAAG,mBAAmB,IAAI,SAAS,EAAE,CAAC;gBAC1D,IAAI,CAAC;oBACJ,MAAM,WAAW,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACtD,OAAO,CAAC,GAAG,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;gBACrD,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,WAAW,CAAC,SAAS,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;oBAC7D,OAAO,CAAC,GAAG,CAAC,8BAA8B,SAAS,EAAE,CAAC,CAAC;gBACxD,CAAC;gBAED,gCAAgC;gBAChC,MAAM,eAAe,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;qBACzC,MAAM,CAAC,cAAc,EAAE,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;qBACxD,QAAQ,EAAE,CAAC;gBACb,MAAM,YAAY,GAAG,GAAG,oBAAoB,IAAI,SAAS,EAAE,CAAC;gBAC5D,MAAM,WAAW,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;gBACvE,OAAO,CAAC,GAAG,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC;gBAExD,mBAAmB;gBACnB,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;oBACzB,IAAI,EAAE;wBACL,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;wBACpB,IAAI,EAAE,SAAS;wBACf,WAAW;wBACX,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI;wBAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,IAAI;wBAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;wBAC/B,MAAM;wBACN,SAAS,EAAE,IAAI,IAAI,EAAE;qBACrB;iBACD,CAAC,CAAC;gBACH,aAAa,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC7C,UAAU,EAAE,CAAC;YACd,CAAC;QACF,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;IACpD,CAAC;IAED,KAAK,UAAU,WAAW;QACzB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;YAClD,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;YAC1C,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CACrC,CAAC,CAA+B,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAC3D,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC3B,OAAO,CAAC,CAAC;QACV,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACJ,IAAI,MAAM,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACP,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;oBACvD,MAAM,WAAW,CAAC,YAAY,CAC7B,UAAU,EACV,GAAG,mBAAmB,IAAI,KAAK,CAAC,IAAI,EAAE,CACtC,CAAC;oBACF,MAAM,WAAW,CAAC,YAAY,CAC7B,UAAU,EACV,GAAG,oBAAoB,IAAI,KAAK,CAAC,IAAI,EAAE,CACvC,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvC,CAAC;gBACD,YAAY,EAAE,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,IAAI,IAAI,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;QACF,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,MAAM,YAAY,EAAE,CAAC;QACzE,MAAM,YAAY,GAAG,MAAM,WAAW,EAAE,CAAC;QACzC,OAAO,CAAC,GAAG,CACV,eAAe,aAAa,YAAY,YAAY,UAAU,YAAY,WAAW,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAChI,CAAC;QACF,MAAM,mBAAmB,CAAC,UAAU,CAAC,GAAG,WAAW,YAAY,EAAE;YAChE,MAAM,EAAE,WAAW;SACnB,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,mBAAmB,CAAC,WAAW,CAAC,GAAG,WAAW,YAAY,KAAK,EAAE,EAAE;YACxE,MAAM,EAAE,WAAW;SACnB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export declare const INGEST_CONFIG: {
|
|
2
2
|
readonly paths: {
|
|
3
|
-
readonly
|
|
4
|
-
readonly
|
|
3
|
+
readonly notes: "markdown/note/**/*.md";
|
|
4
|
+
readonly books: "markdown/book/**/*.md";
|
|
5
|
+
readonly articles: "json/article/**/*.json";
|
|
5
6
|
};
|
|
6
7
|
};
|
|
7
8
|
//# sourceMappingURL=ingest-config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-config.d.ts","sourceRoot":"","sources":["../../src/rag/ingest-config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa
|
|
1
|
+
{"version":3,"file":"ingest-config.d.ts","sourceRoot":"","sources":["../../src/rag/ingest-config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa;;;;;;CAMhB,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export const INGEST_CONFIG = {
|
|
2
2
|
paths: {
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
notes: "markdown/note/**/*.md",
|
|
4
|
+
books: "markdown/book/**/*.md",
|
|
5
|
+
articles: "json/article/**/*.json",
|
|
5
6
|
},
|
|
6
7
|
};
|
|
7
8
|
//# sourceMappingURL=ingest-config.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-config.js","sourceRoot":"","sources":["../../src/rag/ingest-config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,KAAK,EAAE;QACN,
|
|
1
|
+
{"version":3,"file":"ingest-config.js","sourceRoot":"","sources":["../../src/rag/ingest-config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,KAAK,EAAE;QACN,KAAK,EAAE,uBAAuB;QAC9B,KAAK,EAAE,uBAAuB;QAC9B,QAAQ,EAAE,wBAAwB;KAClC;CACQ,CAAC"}
|
package/dist/rag/ingest.js
CHANGED
|
@@ -8,24 +8,24 @@ import { ensureCollection, getCollectionStats, } from "@s-hirano-ist/s-search/qd
|
|
|
8
8
|
import { glob } from "glob";
|
|
9
9
|
import { INGEST_CONFIG } from "./ingest-config.js";
|
|
10
10
|
/**
|
|
11
|
-
* List all files to process
|
|
11
|
+
* List all files to process with content type information
|
|
12
12
|
*/
|
|
13
13
|
async function listFiles() {
|
|
14
14
|
const files = [];
|
|
15
|
-
// JSON
|
|
16
|
-
const
|
|
17
|
-
for (const path of
|
|
18
|
-
files.push({ path, type: "json" });
|
|
15
|
+
// Articles (JSON)
|
|
16
|
+
const articleFiles = await glob(INGEST_CONFIG.paths.articles);
|
|
17
|
+
for (const path of articleFiles) {
|
|
18
|
+
files.push({ path, type: "json", contentType: "articles" });
|
|
19
19
|
}
|
|
20
|
-
//
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
:
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}
|
|
20
|
+
// Notes (Markdown)
|
|
21
|
+
const noteFiles = await glob(INGEST_CONFIG.paths.notes);
|
|
22
|
+
for (const path of noteFiles) {
|
|
23
|
+
files.push({ path, type: "markdown", contentType: "notes" });
|
|
24
|
+
}
|
|
25
|
+
// Books (Markdown)
|
|
26
|
+
const bookFiles = await glob(INGEST_CONFIG.paths.books);
|
|
27
|
+
for (const path of bookFiles) {
|
|
28
|
+
files.push({ path, type: "markdown", contentType: "books" });
|
|
29
29
|
}
|
|
30
30
|
return files;
|
|
31
31
|
}
|
|
@@ -37,23 +37,30 @@ function parseFile(file) {
|
|
|
37
37
|
if (file.type === "json") {
|
|
38
38
|
return parseJsonArticle(file.path, content);
|
|
39
39
|
}
|
|
40
|
-
return parseMarkdown(file.path, content);
|
|
40
|
+
return parseMarkdown(file.path, content, file.contentType);
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
43
43
|
* CLI entry point: list files, parse into chunks, delegate to ingestChunks
|
|
44
44
|
*/
|
|
45
|
-
async function ingest() {
|
|
45
|
+
async function ingest(force) {
|
|
46
46
|
console.log("Starting ingest...\n");
|
|
47
47
|
// Ensure collection exists
|
|
48
48
|
await ensureCollection();
|
|
49
49
|
// Get initial stats
|
|
50
50
|
const initialStats = await getCollectionStats();
|
|
51
51
|
console.log(`Initial points count: ${initialStats.pointsCount}\n`);
|
|
52
|
+
if (force) {
|
|
53
|
+
console.log("Force mode enabled: skipping change detection\n");
|
|
54
|
+
}
|
|
52
55
|
// List all files
|
|
53
56
|
const files = await listFiles();
|
|
57
|
+
const articleCount = files.filter((f) => f.contentType === "articles").length;
|
|
58
|
+
const noteCount = files.filter((f) => f.contentType === "notes").length;
|
|
59
|
+
const bookCount = files.filter((f) => f.contentType === "books").length;
|
|
54
60
|
console.log(`Found ${files.length} files to process`);
|
|
55
|
-
console.log(` -
|
|
56
|
-
console.log(` -
|
|
61
|
+
console.log(` - Articles: ${articleCount}`);
|
|
62
|
+
console.log(` - Notes: ${noteCount}`);
|
|
63
|
+
console.log(` - Books: ${bookCount}\n`);
|
|
57
64
|
// Parse all files into chunks
|
|
58
65
|
console.log("Parsing files...");
|
|
59
66
|
const allChunks = [];
|
|
@@ -76,7 +83,7 @@ async function ingest() {
|
|
|
76
83
|
}).embedBatch
|
|
77
84
|
: undefined;
|
|
78
85
|
// Delegate to core ingest logic
|
|
79
|
-
const result = await ingestChunks(allChunks, { embedBatchFn });
|
|
86
|
+
const result = await ingestChunks(allChunks, { embedBatchFn, force });
|
|
80
87
|
// Get final stats
|
|
81
88
|
const finalStats = await getCollectionStats();
|
|
82
89
|
console.log(`\nFinal points count: ${finalStats.pointsCount}`);
|
|
@@ -94,8 +101,9 @@ async function main() {
|
|
|
94
101
|
if (!env.CF_ACCESS_CLIENT_ID || !env.CF_ACCESS_CLIENT_SECRET) {
|
|
95
102
|
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET environment variables are required.");
|
|
96
103
|
}
|
|
104
|
+
const force = process.argv.includes("--force");
|
|
97
105
|
try {
|
|
98
|
-
await ingest();
|
|
106
|
+
await ingest(force);
|
|
99
107
|
}
|
|
100
108
|
catch (error) {
|
|
101
109
|
console.error("❌ エラーが発生しました:", error);
|
package/dist/rag/ingest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EACN,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EACN,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAQhD;;GAEG;AACH,KAAK,UAAU,SAAS;IACvB,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,kBAAkB;IAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,IAAc;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,KAAc;IACnC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAEpC,2BAA2B;IAC3B,MAAM,gBAAgB,EAAE,CAAC;IAEzB,oBAAoB;IACpB,MAAM,YAAY,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;IAEnE,IAAI,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAChE,CAAC;IAED,iBAAiB;IACjB,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,IAAI,CAAC,CAAC;IAEzC,8BAA8B;IAC9B,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,MAAM,SAAS,GAAoB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;IAED,iEAAiE;IACjE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB;QACjD,CAAC,CAAC,qBAAqB,CAAC;YACtB,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;YACrC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;YAC3C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE;YACvD,oBAAoB,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,EAAE;SAC/D,CAAC,CAAC,UAAU;QACd,CAAC,CAAC,SAAS,CAAC;IAEb,gCAAgC;IAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAEtE,kBAAkB;IAClB,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CACV,mCAAmC,MAAM,CAAC,aAAa,aAAa,MAAM,CAAC,aAAa,WAAW,CACnG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;QAClC,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB;QACpD,uBAAuB,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB;KACnD,CAAC;IAEX,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CACd,qFAAqF,CACrF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE/C,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/rag/search.js
CHANGED
|
@@ -17,13 +17,18 @@ function parseArgs() {
|
|
|
17
17
|
else if (arg === "--heading" && i + 1 < args.length) {
|
|
18
18
|
options.heading = args[++i];
|
|
19
19
|
}
|
|
20
|
+
else if (arg === "--content-type" && i + 1 < args.length) {
|
|
21
|
+
const value = args[++i];
|
|
22
|
+
const types = value.split(",");
|
|
23
|
+
options.contentType = types.length === 1 ? types[0] : types;
|
|
24
|
+
}
|
|
20
25
|
else {
|
|
21
26
|
positional.push(arg);
|
|
22
27
|
}
|
|
23
28
|
}
|
|
24
29
|
const query = positional.join(" ");
|
|
25
30
|
if (!query) {
|
|
26
|
-
console.error("Usage: rag-search <query> [--top-k N] [--type markdown_note|bookmark_json] [--heading HEADING]");
|
|
31
|
+
console.error("Usage: rag-search <query> [--top-k N] [--type markdown_note|bookmark_json] [--heading HEADING] [--content-type articles|books|notes]");
|
|
27
32
|
process.exit(1);
|
|
28
33
|
}
|
|
29
34
|
return { query, options };
|
|
@@ -32,7 +37,11 @@ async function searchContent(query, options = {}) {
|
|
|
32
37
|
const queryVector = await embed(query, true);
|
|
33
38
|
const results = await qdrantSearch(queryVector, {
|
|
34
39
|
topK: options.topK,
|
|
35
|
-
filter: {
|
|
40
|
+
filter: {
|
|
41
|
+
type: options.type,
|
|
42
|
+
top_heading: options.heading,
|
|
43
|
+
content_type: options.contentType,
|
|
44
|
+
},
|
|
36
45
|
});
|
|
37
46
|
return { results, query, totalResults: results.length };
|
|
38
47
|
}
|
|
@@ -45,6 +54,8 @@ async function search() {
|
|
|
45
54
|
console.log(` type: ${options.type}`);
|
|
46
55
|
if (options.heading)
|
|
47
56
|
console.log(` heading: ${options.heading}`);
|
|
57
|
+
if (options.contentType)
|
|
58
|
+
console.log(` content-type: ${Array.isArray(options.contentType) ? options.contentType.join(",") : options.contentType}`);
|
|
48
59
|
console.log();
|
|
49
60
|
await ensureCollection();
|
|
50
61
|
const response = await searchContent(query, options);
|
|
@@ -55,6 +66,7 @@ async function search() {
|
|
|
55
66
|
if (result.url)
|
|
56
67
|
console.log(`URL: ${result.url}`);
|
|
57
68
|
console.log(`Type: ${result.type}`);
|
|
69
|
+
console.log(`Content-Type: ${result.content_type}`);
|
|
58
70
|
console.log(`Heading: ${result.heading_path.join(" > ")}`);
|
|
59
71
|
console.log(`\n${result.text}\n`);
|
|
60
72
|
}
|
package/dist/rag/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/rag/search.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AAEvB,OAAO,EAAE,KAAK,EAAE,MAAM,kCAAkC,CAAC;AACzD,OAAO,EACN,gBAAgB,EAChB,MAAM,IAAI,YAAY,GACtB,MAAM,sCAAsC,CAAC;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/rag/search.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AAEvB,OAAO,EAAE,KAAK,EAAE,MAAM,kCAAkC,CAAC;AACzD,OAAO,EACN,gBAAgB,EAChB,MAAM,IAAI,YAAY,GACtB,MAAM,sCAAsC,CAAC;AAS9C,SAAS,SAAS;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAA0B,CAAC;QACnD,CAAC;aAAM,IAAI,GAAG,KAAK,WAAW,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,KAAK,gBAAgB,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;YAChD,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC7D,CAAC;aAAM,CAAC;YACP,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CACZ,sIAAsI,CACtI,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,aAAa,CAC3B,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,WAAW,EAAE;QAC/C,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE;YACP,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,YAAY,EAAE,OAAO,CAAC,WAAW;SACjC;KACD,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,MAAM;IACpB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;IAEvC,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,IAAI,OAAO,CAAC,WAAW;QACtB,OAAO,CAAC,GAAG,CACV,mBAAmB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAC7G,CAAC;IACH,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,MAAM,gBAAgB,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAErD,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,CAAC,YAAY,aAAa,CAAC,CAAC;IAEzD,KAAK,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACtC,IAAI,MAAM,CAAC,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;IACnC,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;KACzB,CAAC;IAEX,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,MAAM,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -2,7 +2,39 @@
|
|
|
2
2
|
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
5
|
+
import iconv from "iconv-lite";
|
|
5
6
|
import { JSDOM, VirtualConsole } from "jsdom";
|
|
7
|
+
function normalizeCharset(charset) {
|
|
8
|
+
const normalized = charset.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
9
|
+
const mapping = {
|
|
10
|
+
shiftjis: "Shift_JIS",
|
|
11
|
+
sjis: "Shift_JIS",
|
|
12
|
+
xsjis: "Shift_JIS",
|
|
13
|
+
eucjp: "EUC-JP",
|
|
14
|
+
xeucjp: "EUC-JP",
|
|
15
|
+
};
|
|
16
|
+
return mapping[normalized] || charset;
|
|
17
|
+
}
|
|
18
|
+
function detectCharset(headers, buffer) {
|
|
19
|
+
// 1. Content-Type ヘッダーから検出
|
|
20
|
+
const contentType = headers.get("content-type") || "";
|
|
21
|
+
const headerMatch = contentType.match(/charset=([^\s;]+)/i);
|
|
22
|
+
if (headerMatch)
|
|
23
|
+
return normalizeCharset(headerMatch[1]);
|
|
24
|
+
// 2. HTML meta タグから検出（ASCII範囲で仮デコード）
|
|
25
|
+
const preview = buffer
|
|
26
|
+
.subarray(0, Math.min(4096, buffer.length))
|
|
27
|
+
.toString("ascii");
|
|
28
|
+
// <meta charset="...">
|
|
29
|
+
const metaCharset = preview.match(/<meta\s{1,200}charset=["']?([^"'\s>]+)/i);
|
|
30
|
+
if (metaCharset)
|
|
31
|
+
return normalizeCharset(metaCharset[1]);
|
|
32
|
+
// <meta http-equiv="Content-Type" content="...; charset=...">
|
|
33
|
+
const metaHttpEquiv = preview.match(/<meta[^>]{1,500}http-equiv=["']?Content-Type["']?[^>]{1,500}content=["'][^"']{0,500}charset=([^"'\s;]+)/i);
|
|
34
|
+
if (metaHttpEquiv)
|
|
35
|
+
return normalizeCharset(metaHttpEquiv[1]);
|
|
36
|
+
return "utf-8";
|
|
37
|
+
}
|
|
6
38
|
// REF: https://zenn.dev/littleforest/articles/scrape-og-tags
|
|
7
39
|
function extractOgpData(metaElements) {
|
|
8
40
|
return metaElements
|
|
@@ -19,9 +51,20 @@ function extractOgpData(metaElements) {
|
|
|
19
51
|
async function getOgTags(url) {
|
|
20
52
|
try {
|
|
21
53
|
console.log(`Fetching OG tags for: ${url}`);
|
|
54
|
+
const response = await fetch(url, {
|
|
55
|
+
headers: {
|
|
56
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
if (!response.ok) {
|
|
60
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
61
|
+
}
|
|
62
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
63
|
+
const charset = detectCharset(response.headers, buffer);
|
|
64
|
+
const html = iconv.decode(buffer, charset);
|
|
22
65
|
// REF: https://github.com/jsdom/jsdom#virtual-consoles
|
|
23
66
|
const virtualConsole = new VirtualConsole();
|
|
24
|
-
const dom =
|
|
67
|
+
const dom = new JSDOM(html, { virtualConsole });
|
|
25
68
|
virtualConsole.on("error", () => { });
|
|
26
69
|
virtualConsole.on("warn", () => { });
|
|
27
70
|
virtualConsole.on("info", () => { });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAkB9C,6DAA6D;AAC7D,SAAS,cAAc,CAAC,YAA+B;IACtD,OAAO,YAAY;SACjB,MAAM,CAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;SAC9D,MAAM,CACN,CAAC,QAAgC,EAAE,OAAgB,EAAE,EAAE;QACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;YAAE,OAAO,QAAQ,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;QAC7B,OAAO,QAAQ,CAAC;IACjB,CAAC,EACD,EAA4B,CAC5B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,GAAW;IAEX,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAE5C,
|
|
1
|
+
{"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAkB9C,SAAS,gBAAgB,CAAC,OAAe;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACnE,MAAM,OAAO,GAA2B;QACvC,QAAQ,EAAE,WAAW;QACrB,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,QAAQ;KAChB,CAAC;IACF,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC;AACvC,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,MAAc;IACtD,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC5D,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,qCAAqC;IACrC,MAAM,OAAO,GAAG,MAAM;SACpB,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;SAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEpB,uBAAuB;IACvB,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7E,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAClC,0GAA0G,CAC1G,CAAC;IACF,IAAI,aAAa;QAAE,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,6DAA6D;AAC7D,SAAS,cAAc,CAAC,YAA+B;IACtD,OAAO,YAAY;SACjB,MAAM,CAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;SAC9D,MAAM,CACN,CAAC,QAAgC,EAAE,OAAgB,EAAE,EAAE;QACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;YAAE,OAAO,QAAQ,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;QAC7B,OAAO,QAAQ,CAAC;IACjB,CAAC,EACD,EAA4B,CAC5B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,GAAW;IAEX,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3C,uDAAuD;QACvD,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAChD,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACpC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,CAAC;QAEjD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,GAAG,CAAC,CAAC;QACpD,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACjD,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,GAAiB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE1D,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,KAAK,CAAC,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEjC,gCAAgC;YAChC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxE,SAAS;YACV,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEzC,0CAA0C;YAC1C,IAAI,MAAM,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;;gBACtD,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;;gBAC7C,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,aAAa;gBAAE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;;gBAC/D,OAAO,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE7D,kDAAkD;YAClD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,qBAAqB;QACrB,aAAa,CACZ,QAAQ,EACR,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAC9C,OAAO,CACP,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1D,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CACtB,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,MAAM,mBAAmB,CAAC,UAAU,CAAC,gCAAgC,EAAE;YACtE,MAAM,EAAE,sBAAsB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,gCAAgC,KAAK,EAAE,EACvC;YACC,MAAM,EAAE,sBAAsB;SAC9B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@s-hirano-ist/s-scripts",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.15.0",
|
|
4
4
|
"description": "CLI scripts for s-private data operations",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -45,14 +45,14 @@
|
|
|
45
45
|
"dotenv": "16.5.0",
|
|
46
46
|
"glob": "13.0.0",
|
|
47
47
|
"jsdom": "27.4.0",
|
|
48
|
-
"minio": "8.0.6",
|
|
49
48
|
"sharp": "0.33.5",
|
|
50
49
|
"iconv-lite": "0.7.2",
|
|
51
50
|
"turndown": "7.2.2",
|
|
52
|
-
"@s-hirano-ist/s-core": "1.
|
|
53
|
-
"@s-hirano-ist/s-
|
|
54
|
-
"@s-hirano-ist/s-
|
|
55
|
-
"@s-hirano-ist/s-
|
|
51
|
+
"@s-hirano-ist/s-core": "1.15.0",
|
|
52
|
+
"@s-hirano-ist/s-database": "1.15.0",
|
|
53
|
+
"@s-hirano-ist/s-notification": "1.15.0",
|
|
54
|
+
"@s-hirano-ist/s-search": "1.15.0",
|
|
55
|
+
"@s-hirano-ist/s-storage": "1.15.0"
|
|
56
56
|
},
|
|
57
57
|
"devDependencies": {
|
|
58
58
|
"@types/jsdom": "27.0.0",
|