@s-hirano-ist/s-scripts 1.23.0 → 1.23.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,11 @@
1
1
  #!/usr/bin/env node
2
2
  /**
3
- * markdown/book/*.md の frontmatter のうち google* が欠落しているものについて、
4
- * Google Books API を呼び出して frontmatter を補完するスクリプト。
3
+ * markdown/book/*.md の frontmatter について、毎回 Google Books API を呼び出して
4
+ * google* および title を最新状態に更新するスクリプト。
5
5
  *
6
6
  * API が title を返した場合は frontmatter.title を上書きする。
7
7
  * notFound (API が volume を返さない) の場合は手動入力 title を保持する。
8
+ * markdown 本文 (h1 含む) には触れないため、自己入力の h1 はそのまま維持される。
8
9
  */
9
10
  import { readFile, writeFile } from "node:fs/promises";
10
11
  import { basename, join } from "node:path";
@@ -35,7 +36,6 @@ async function main() {
35
36
  const files = await glob(`${bookDir}/*.md`);
36
37
  console.log(`📁 ${files.length} 件の markdown を検出しました。${dryRun ? " (dry-run)" : ""}`);
37
38
  let enrichedCount = 0;
38
- let skippedCount = 0;
39
39
  let errorCount = 0;
40
40
  for (const filePath of files.sort()) {
41
41
  const fileName = basename(filePath);
@@ -44,11 +44,6 @@ async function main() {
44
44
  const raw = await readFile(filePath, "utf8");
45
45
  const parsed = matter(raw);
46
46
  const data = parsed.data;
47
- // googleImgSrc が既に埋まっているならスキップ (enrich 実行済みの判定キー)
48
- if (data.googleImgSrc != null && data.googleImgSrc !== "") {
49
- skippedCount++;
50
- continue;
51
- }
52
47
  console.log(`🔍 ${fileName}: Google Books API 問い合わせ中...`);
53
48
  const book = await api.volumes.list({ q: `isbn:${isbn}` });
54
49
  await sleep(API_INTERVAL_MS);
@@ -87,7 +82,7 @@ async function main() {
87
82
  errorCount++;
88
83
  }
89
84
  }
90
- console.log(`\n📊 結果: 補完 ${enrichedCount} 件, スキップ ${skippedCount} 件, エラー ${errorCount} 件${dryRun ? " (dry-run)" : ""}`);
85
+ console.log(`\n📊 結果: 更新 ${enrichedCount} 件, エラー ${errorCount} 件${dryRun ? " (dry-run)" : ""}`);
91
86
  }
92
87
  main().catch((error) => {
93
88
  console.error("❌ エラーが発生しました:", error);
@@ -1 +1 @@
1
- {"version":3,"file":"enrich-books.js","sourceRoot":"","sources":["../src/enrich-books.ts"],"names":[],"mappings":";AACA;;;;;;GAMG;AACH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,KAAK,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,SAAS,CAAC;AAE3B,MAAM,UAAU,GAAG,mCAAmC,CAAC;AACvD,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAClD,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,eAAe,CAAC,IAA6B;IACrD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACtB,SAAS,EAAE,CAAC,CAAC;QACb,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,IAAI;KACZ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAC3C,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEnD,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CACV,MAAM,KAAK,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CACtE,CAAC;IAEF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEvC,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAA+B,CAAC;YAEpD,kDAAkD;YAClD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,KAAK,EAAE,EAAE,CAAC;gBAC3D,YAAY,EAAE,CAAC;gBACf,SAAS;YACV,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,QAAQ,8BAA8B,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC;YAChD,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,UAAU,CAAC,CAAC,OAAO,CACxE,SAAS,EACT,UAAU,CACV,CAAC;YACF,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,OAAO,CAC7D,SAAS,EACT,UAAU,CACV,CAAC;YAEF,MAAM,IAAI,GAA4B;gBACrC,GAAG,IAAI;gBACP,cAAc,EAAE,MAAM,EAAE,QAAQ,IAAI,EAAE;gBACtC,aAAa,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE;gBACpC,iBAAiB,EAAE,MAAM,EAAE,WAAW,IAAI,gBAAgB;gBAC1D,YAAY,EAAE,WAAW;gBACzB,UAAU,EAAE,SAAS;aACrB,CAAC;YAEF,+CAA+C;YAC/C,iCAAiC;YACjC,IAAI,OAAO,MAAM,EAAE,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;gBAC9D,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,QAAQ,eAAe,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAEjE,IAAI,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACP,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,EAAE,KAAK,IAAI,mBAAmB,EAAE,CAAC,CAAC;YAC5D,CAAC;YACD,aAAa,EAAE,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,KAAK,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;YACvC,UAAU,EAAE,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CACV,eAAe,aAAa,YAAY,YAAY,WAAW,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1G,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"enrich-books.js","sourceRoot":"","sources":["../src/enrich-books.ts"],"names":[],"mappings":";AACA;;;;;;;GAOG;AACH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,KAAK,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,SAAS,CAAC;AAE3B,MAAM,UAAU,GAAG,mCAAmC,CAAC;AACvD,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAClD,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,eAAe,CAAC,IAA6B;IACrD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACtB,SAAS,EAAE,CAAC,CAAC;QACb,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,IAAI;KACZ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAC3C,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEnD,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CACV,MAAM,KAAK,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CACtE,CAAC;IAEF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEvC,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAA+B,CAAC;YAEpD,OAAO,CAAC,GAAG,CAAC,MAAM,QAAQ,8BAA8B,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC;YAChD,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,UAAU,CAAC,CAAC,OAAO,CACxE,SAAS,EACT,UAAU,CACV,CAAC;YACF,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,OAAO,CAC7D,SAAS,EACT,UAAU,CACV,CAAC;YAEF,MAAM,IAAI,GAA4B;gBACrC,GAAG,IAAI;gBACP,cAAc,EAAE,MAAM,EAAE,QAAQ,IAAI,EAAE;gBACtC,aAAa,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE;gBACpC,iBAAiB,EAAE,MAAM,EAAE,WAAW,IAAI,gBAAgB;gBAC1D,YAAY,EAAE,WAAW;gBACzB,UAAU,EAAE,SAAS;aACrB,CAAC;YAEF,+CAA+C;YAC/C,iCAAiC;YACjC,IAAI,OAAO,MAAM,EAAE,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;gBAC9D,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,QAAQ,eAAe,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAEjE,IAAI,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACP,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,EAAE,KAAK,IAAI,mBAAmB,EAAE,CAAC,CAAC;YAC5D,CAAC;YACD,aAAa,EAAE,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,KAAK,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;YACvC,UAAU,EAAE,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CACV,eAAe,aAAa,WAAW,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  import { readFileSync } from "node:fs";
3
3
  import { parseJsonArticle, parseMarkdown, } from "@s-hirano-ist/s-search/chunker";
4
- import { ingestChunks } from "@s-hirano-ist/s-search/ingest";
4
+ import { ingestChunks, pruneOrphans } from "@s-hirano-ist/s-search/ingest";
5
5
  import { ensureCollection, getCollectionStats, } from "@s-hirano-ist/s-search/qdrant-client";
6
6
  import { glob } from "glob";
7
7
  import { INGEST_CONFIG } from "./ingest-config.js";
@@ -40,17 +40,19 @@ function parseFile(file) {
40
40
  /**
41
41
  * CLI entry point: list files, parse into chunks, delegate to ingestChunks
42
42
  */
43
- async function ingest(force) {
43
+ async function ingest(flags) {
44
44
  console.log("Starting ingest...\n");
45
- // Ensure collection exists
46
- await ensureCollection();
47
- // Get initial stats
48
- const initialStats = await getCollectionStats();
49
- console.log(`Initial points count: ${initialStats.pointsCount}\n`);
50
- if (force) {
45
+ if (!flags.dryRun) {
46
+ await ensureCollection();
47
+ const initialStats = await getCollectionStats();
48
+ console.log(`Initial points count: ${initialStats.pointsCount}\n`);
49
+ }
50
+ else {
51
+ console.log("Dry-run mode: no writes to Qdrant\n");
52
+ }
53
+ if (flags.force) {
51
54
  console.log("Force mode enabled: skipping change detection\n");
52
55
  }
53
- // List all files
54
56
  const files = await listFiles();
55
57
  const articleCount = files.filter((f) => f.contentType === "articles").length;
56
58
  const noteCount = files.filter((f) => f.contentType === "notes").length;
@@ -59,7 +61,6 @@ async function ingest(force) {
59
61
  console.log(` - Articles: ${articleCount}`);
60
62
  console.log(` - Notes: ${noteCount}`);
61
63
  console.log(` - Books: ${bookCount}\n`);
62
- // Parse all files into chunks
63
64
  console.log("Parsing files...");
64
65
  const allChunks = [];
65
66
  for (const file of files) {
@@ -71,28 +72,36 @@ async function ingest(force) {
71
72
  console.error(`Error parsing ${file.path}:`, error);
72
73
  }
73
74
  }
74
- // Delegate to core ingest logic (Qdrant Inference handles embedding)
75
- const result = await ingestChunks(allChunks, { force });
76
- // Get final stats
75
+ console.log(`Total chunks parsed: ${allChunks.length}`);
76
+ if (flags.dryRun) {
77
+ console.log("Dry-run complete. Skipping upsert and prune.");
78
+ return;
79
+ }
80
+ const result = await ingestChunks(allChunks, { force: flags.force });
81
+ const pruneResult = await pruneOrphans(allChunks.map((c) => c.chunk_id));
77
82
  const finalStats = await getCollectionStats();
78
83
  console.log(`\nFinal points count: ${finalStats.pointsCount}`);
79
- console.log(`Ingest completed successfully! (${result.changedChunks} changed, ${result.skippedChunks} skipped)`);
84
+ console.log(`Ingest completed! (${result.changedChunks} changed, ${result.skippedChunks} skipped, ${pruneResult.deletedCount} pruned)`);
80
85
  }
81
86
  async function main() {
82
87
  if (!process.env.QDRANT_URL) {
83
- throw new Error("QDRANT_URL environment variable is required.");
88
+ console.error("QDRANT_URL environment variable is required.");
89
+ process.exit(1);
84
90
  }
85
- const force = process.argv.includes("--force");
91
+ const flags = {
92
+ force: process.argv.includes("--force"),
93
+ dryRun: process.argv.includes("--dry-run"),
94
+ };
86
95
  try {
87
- await ingest(force);
96
+ await ingest(flags);
88
97
  }
89
98
  catch (error) {
90
99
  console.error("❌ エラーが発生しました:", error);
91
- process.exit(1);
100
+ process.exit(2);
92
101
  }
93
102
  }
94
103
  main().catch((error) => {
95
104
  console.error(error);
96
- process.exit(1);
105
+ process.exit(2);
97
106
  });
98
107
  //# sourceMappingURL=ingest.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EACN,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAQnD;;GAEG;AACH,KAAK,UAAU,SAAS;IACvB,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,kBAAkB;IAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,IAAc;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,KAAc;IACnC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAEpC,2BAA2B;IAC3B,MAAM,gBAAgB,EAAE,CAAC;IAEzB,oBAAoB;IACpB,MAAM,YAAY,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;IAEnE,IAAI,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAChE,CAAC;IAED,iBAAiB;IACjB,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,IAAI,CAAC,CAAC;IAEzC,8BAA8B;IAC9B,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,MAAM,SAAS,GAAoB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;IAED,qEAAqE;IACrE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,kBAAkB;IAClB,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CACV,mCAAmC,MAAM,CAAC,aAAa,aAAa,MAAM,CAAC,aAAa,WAAW,CACnG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE/C,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EACN,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAQnD;;GAEG;AACH,KAAK,UAAU,SAAS;IACvB,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,kBAAkB;IAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,IAAc;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AAC5D,CAAC;AAOD;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,KAAkB;IACvC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAEpC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,gBAAgB,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,IAAI,CAAC,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,MAAM,SAAS,GAAoB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;QAC5D,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEzE,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CACV,sBAAsB,MAAM,CAAC,aAAa,aAAa,MAAM,CAAC,aAAa,aAAa,WAAW,CAAC,YAAY,UAAU,CAC1H,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAgB;QAC1B,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACvC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;KAC1C,CAAC;IAEF,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@s-hirano-ist/s-scripts",
3
- "version": "1.23.0",
3
+ "version": "1.23.2",
4
4
  "description": "CLI scripts for s-private data operations",
5
5
  "type": "module",
6
6
  "bin": {
@@ -58,11 +58,11 @@
58
58
  "js-yaml": "4.1.1",
59
59
  "sharp": "0.34.5",
60
60
  "turndown": "7.2.4",
61
+ "@s-hirano-ist/s-core": "1.20.0",
61
62
  "@s-hirano-ist/s-database": "1.19.0",
62
63
  "@s-hirano-ist/s-notification": "1.18.4",
63
- "@s-hirano-ist/s-search": "1.18.8",
64
64
  "@s-hirano-ist/s-storage": "1.18.5",
65
- "@s-hirano-ist/s-core": "1.20.0"
65
+ "@s-hirano-ist/s-search": "1.18.9"
66
66
  },
67
67
  "devDependencies": {
68
68
  "@types/js-yaml": "4.0.9",