@s-hirano-ist/s-scripts 4.1.0 → 4.3.0

package/dist/fix-book-frontmatter.js

@@ -6,6 +6,10 @@ import {
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import {
+  reportCliError,
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/fix-book-frontmatter.ts
@@ -94,6 +98,7 @@ ${body}`;
     }
     return "fixed";
   } catch (error) {
+    reportCliError(error, "fix-book-frontmatter");
     console.error(`\u274C ${fileName}: ${String(error)}`);
     return "error";
   }
@@ -127,7 +132,4 @@ async function main() {
 \u{1F4CA} \u7D50\u679C: \u4FEE\u6B63 ${fixedCount} \u4EF6, \u30B9\u30AD\u30C3\u30D7 ${skippedCount} \u4EF6, \u30A8\u30E9\u30FC ${errorCount} \u4EF6${dryRun ? " (dry-run)" : ""}`
   );
 }
-main().catch((error) => {
-  console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
-  process.exit(1);
-});
+await runCli("fix-book-frontmatter", main);

package/dist/fix-image-formats.js

@@ -6,6 +6,10 @@ import {
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import {
+  reportCliError,
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/fix-image-formats.ts
@@ -64,6 +68,7 @@ async function processDirectory(directory, label, dryRun) {
       console.log(`\u2705 \u5909\u63DB\u5B8C\u4E86: ${fileName} \u2192 ${basename(outputPath)}`);
       converted++;
     } catch (error) {
+      reportCliError(error, "fix-image-formats");
       console.error(`\u274C ${fileName}: ${String(error)}`);
       errors++;
     }
@@ -87,10 +92,7 @@ async function main() {
 \u{1F4CA} \u5408\u8A08: \u5909\u63DB ${converted} \u4EF6, \u30B9\u30AD\u30C3\u30D7 ${skipped} \u4EF6, \u30A8\u30E9\u30FC ${errors} \u4EF6${dryRun ? " (dry-run)" : ""}`
   );
   if (errors > 0) {
-    process.exit(1);
+    process.exitCode = 1;
   }
 }
-main().catch((error) => {
-  console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
-  process.exit(1);
-});
+await runCli("fix-image-formats", main);

package/dist/ingest-articles.js

@@ -5,14 +5,16 @@ import {
 } from "./chunk-IYGLKY27.js";
 import {
   runContentImport
-} from "./chunk-2PZXXOYG.js";
+} from "./chunk-GBQOCN2C.js";
 import "./chunk-KKZP7ZF3.js";
-import "./chunk-F4J6JBR4.js";
-import "./chunk-DIG75T3J.js";
-import "./chunk-7PVU2Z32.js";
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import "./chunk-MVYAZJB2.js";
+import "./chunk-RLAYINCA.js";
+import {
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/ingest-articles.ts
@@ -33,4 +35,4 @@ async function* source(root) {
       };
   }
 }
-await runContentImport("articles", source);
+await runCli("ingest-articles", () => runContentImport("articles", source));

package/dist/ingest-books.js

@@ -1,19 +1,21 @@
 #!/usr/bin/env node
 import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  parseMarkdownFrontmatter
+} from "./chunk-HDYQV2Y2.js";
 import {
   makeISBN,
   runContentImport
-} from "./chunk-2PZXXOYG.js";
+} from "./chunk-GBQOCN2C.js";
 import "./chunk-KKZP7ZF3.js";
-import "./chunk-F4J6JBR4.js";
-import "./chunk-DIG75T3J.js";
-import "./chunk-7PVU2Z32.js";
-import {
-  parseMarkdownFrontmatter
-} from "./chunk-HDYQV2Y2.js";
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import "./chunk-MVYAZJB2.js";
+import "./chunk-RLAYINCA.js";
+import {
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/ingest-books.ts
@@ -65,4 +67,4 @@ async function* source(root) {
     };
   }
 }
-await runContentImport("books", source);
+await runCli("ingest-books", () => runContentImport("books", source));

package/dist/ingest-images.js

@@ -3,14 +3,16 @@ import { createRequire as __cliCreateRequire } from "node:module"; const require
 import {
   runContentImport,
   sharpImageProcessor
-} from "./chunk-2PZXXOYG.js";
+} from "./chunk-GBQOCN2C.js";
 import "./chunk-KKZP7ZF3.js";
-import "./chunk-F4J6JBR4.js";
-import "./chunk-DIG75T3J.js";
-import "./chunk-7PVU2Z32.js";
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import "./chunk-MVYAZJB2.js";
+import "./chunk-RLAYINCA.js";
+import {
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/ingest-images.ts
@@ -34,4 +36,4 @@ async function* source(root) {
     };
   }
 }
-await runContentImport("images", source);
+await runCli("ingest-images", () => runContentImport("images", source));

package/dist/ingest-notes.js

@@ -1,18 +1,20 @@
 #!/usr/bin/env node
 import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
-import {
-  runContentImport
-} from "./chunk-2PZXXOYG.js";
-import "./chunk-KKZP7ZF3.js";
-import "./chunk-F4J6JBR4.js";
-import "./chunk-DIG75T3J.js";
-import "./chunk-7PVU2Z32.js";
 import {
   parseMarkdownFrontmatter
 } from "./chunk-HDYQV2Y2.js";
+import {
+  runContentImport
+} from "./chunk-GBQOCN2C.js";
+import "./chunk-KKZP7ZF3.js";
 import {
   globPaths
 } from "./chunk-AJMTRCPJ.js";
+import "./chunk-MVYAZJB2.js";
+import "./chunk-RLAYINCA.js";
+import {
+  runCli
+} from "./chunk-4CXH2QIX.js";
 import "./chunk-JZOR5GA2.js";
 
 // src/ingest-notes.ts
@@ -46,4 +48,4 @@ async function* source(root) {
     if (values) yield { values };
   }
 }
-await runContentImport("notes", source);
+await runCli("ingest-notes", () => runContentImport("notes", source));

package/dist/rag/delete-collection.js

@@ -1,37 +1,12 @@
 #!/usr/bin/env node
 import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
 import {
-  deleteCollection,
-  getQdrantClient
-} from "../chunk-J4VLH4VA.js";
+  runCli
+} from "../chunk-4CXH2QIX.js";
 import "../chunk-JZOR5GA2.js";
 
 // src/rag/delete-collection.ts
-async function main() {
-  const collectionName = process.argv[2];
-  if (!collectionName) {
-    console.error(
-      "Usage: rag-delete-collection <collection-name>\nExample: rag-delete-collection knowledge_v1"
-    );
-    process.exit(1);
-  }
-  if (!process.env.QDRANT_URL) {
-    throw new Error("QDRANT_URL environment variable is required.");
-  }
-  const qdrant = getQdrantClient();
-  const collections = await qdrant.getCollections();
-  const exists = collections.collections.some((c) => c.name === collectionName);
-  if (!exists) {
-    console.log(
-      `Collection "${collectionName}" does not exist. Nothing to delete.`
-    );
-    return;
-  }
-  console.log(`Deleting collection "${collectionName}"...`);
-  await deleteCollection(collectionName);
-  console.log(`Collection "${collectionName}" deleted successfully.`);
-}
-main().catch((error) => {
-  console.error("Failed to delete collection:", error);
-  process.exit(1);
+await runCli("rag-delete-collection", async () => {
+  const { main } = await import("../chunk-67LIMNXH.js");
+  await main();
 });

package/dist/rag/ingest.js

@@ -1,386 +1,12 @@
 #!/usr/bin/env node
 import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
 import {
-  parseMarkdownFrontmatter
-} from "../chunk-HDYQV2Y2.js";
-import {
-  globPaths
-} from "../chunk-AJMTRCPJ.js";
-import {
-  RAG_CONFIG,
-  deletePointsByChunkIds,
-  ensureCollection,
-  getCollectionStats,
-  getExistingHashes,
-  listAllChunkIds,
-  upsertPoints
-} from "../chunk-J4VLH4VA.js";
+  runCli
+} from "../chunk-4CXH2QIX.js";
 import "../chunk-JZOR5GA2.js";
 
-// ../search/dist/chunker.js
-import { createHash } from "node:crypto";
-function generateHash(content) {
-  return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-function generateChunkId(docId, index) {
-  return `${docId}#${index}`;
-}
-function parseJsonArticle(filePath, content) {
-  const json = JSON.parse(content);
-  const docId = `file:${filePath}`;
-  const chunks = [];
-  for (let i = 0; i < json.body.length; i++) {
-    const item = json.body[i];
-    const textParts = [];
-    if (item.title)
-      textParts.push(item.title);
-    if (item.ogTitle && item.ogTitle !== item.title)
-      textParts.push(item.ogTitle);
-    if (item.ogDescription)
-      textParts.push(item.ogDescription);
-    if (item.quote)
-      textParts.push(item.quote);
-    const text = textParts.join("\n");
-    if (!text.trim())
-      continue;
-    const chunkId = generateChunkId(docId, i);
-    chunks.push({
-      type: "bookmark_json",
-      content_type: "articles",
-      top_heading: json.heading,
-      doc_id: docId,
-      chunk_id: chunkId,
-      title: item.title || item.ogTitle || "Untitled",
-      url: item.url,
-      heading_path: [json.heading],
-      text,
-      content_hash: generateHash(text)
-    });
-  }
-  return chunks;
-}
-function parseFrontmatter(content) {
-  const parsed = parseMarkdownFrontmatter(content);
-  const frontmatter = {
-    heading: typeof parsed.data.heading === "string" ? parsed.data.heading : "unknown"
-  };
-  if (typeof parsed.data.description === "string") {
-    frontmatter.description = parsed.data.description;
-  }
-  if (typeof parsed.data.draft === "boolean") {
-    frontmatter.draft = parsed.data.draft;
-  }
-  return { frontmatter, body: parsed.content };
-}
-function pushPreambleSection(state) {
-  if (state.preambleContent.trim()) {
-    state.sections.push({
-      headingPath: [],
-      title: "",
-      content: state.preambleContent,
-      level: 0
-    });
-  }
-}
-function handleHeadingLine(state, headingMatch) {
-  if (!state.preambleDone) {
-    state.preambleDone = true;
-    pushPreambleSection(state);
-  }
-  if (state.currentSection?.content.trim()) {
-    state.sections.push(state.currentSection);
-  }
-  const level = headingMatch[1].length;
-  const title = headingMatch[2];
-  const headingStack = state.headingStack;
-  let top = headingStack.at(-1);
-  while (top && top.level >= level) {
-    headingStack.pop();
-    top = headingStack.at(-1);
-  }
-  headingStack.push({ level, title });
-  state.currentSection = {
-    headingPath: headingStack.map((h) => h.title),
-    title,
-    content: "",
-    level
-  };
-}
-function splitMarkdownByHeadings(content) {
-  const state = {
-    sections: [],
-    currentSection: null,
-    headingStack: [],
-    // Preamble buffer (content before first H2/H3)
-    preambleContent: "",
-    preambleDone: false
-  };
-  for (const line of content.split("\n")) {
-    if (/^#\s+\S/u.test(line)) {
-      continue;
-    }
-    const headingMatch = /^(#{2,3})\s+(\S.*)$/u.exec(line);
-    if (headingMatch) {
-      handleHeadingLine(state, headingMatch);
-    } else if (!state.preambleDone) {
-      state.preambleContent += `${line}
-`;
-    } else if (state.currentSection) {
-      state.currentSection.content += `${line}
-`;
-    }
-  }
-  if (!state.preambleDone) {
-    pushPreambleSection(state);
-  }
-  if (state.currentSection?.content.trim()) {
-    state.sections.push(state.currentSection);
-  }
-  return state.sections;
-}
-function splitByParagraphs(text, maxLength) {
-  if (text.length <= maxLength) {
-    return [text];
-  }
-  const paragraphs = text.split(/\n{2,}/u);
-  const chunks = [];
-  let currentChunk = "";
-  for (const para of paragraphs) {
-    if (currentChunk.length + para.length > maxLength && currentChunk) {
-      chunks.push(currentChunk.trim());
-      currentChunk = para;
-    } else {
-      currentChunk += (currentChunk ? "\n\n" : "") + para;
-    }
-  }
-  if (currentChunk.trim()) {
-    chunks.push(currentChunk.trim());
-  }
-  return chunks;
-}
-function parseMarkdownWithDocId(docId, content, contentType = "notes") {
-  const { frontmatter, body } = parseFrontmatter(content);
-  const chunks = [];
-  if (frontmatter.draft) {
-    return [];
-  }
-  const sections = splitMarkdownByHeadings(body);
-  let chunkIndex = 0;
-  for (const section of sections) {
-    const textChunks = splitByParagraphs(section.content, RAG_CONFIG.chunking.maxChunkLength);
-    for (const text of textChunks) {
-      if (!text.trim())
-        continue;
-      const isPreamble = section.level === 0 && section.title === "";
-      const sectionTitle = isPreamble ? frontmatter.description ?? frontmatter.heading : section.title;
-      const fullHeadingPath = isPreamble ? [frontmatter.heading] : [frontmatter.heading, ...section.headingPath];
-      const chunkId = generateChunkId(docId, chunkIndex);
-      chunks.push({
-        type: "markdown_note",
-        content_type: contentType,
-        top_heading: frontmatter.heading,
-        doc_id: docId,
-        chunk_id: chunkId,
-        title: sectionTitle,
-        heading_path: fullHeadingPath,
-        text,
-        content_hash: generateHash(text)
-      });
-      chunkIndex++;
-    }
-  }
-  return chunks;
-}
-function parseMarkdown(filePath, content, contentType = "notes") {
-  return parseMarkdownWithDocId(`file:${filePath}`, content, contentType);
-}
-
-// ../search/dist/ingest.js
-import { setTimeout } from "node:timers/promises";
-var BATCH_SIZE = 20;
-var MAX_RETRIES = 3;
-var RETRY_DELAY_MS = 2e3;
-async function withRetry(fn, retries = MAX_RETRIES) {
-  for (let i = 0; i < retries; i++) {
-    try {
-      return await fn();
-    } catch (error) {
-      if (i === retries - 1)
-        throw error;
-      console.log(`  Retry ${i + 1}/${retries} after error...`);
-      await setTimeout(RETRY_DELAY_MS);
-    }
-  }
-  throw new Error("Unreachable");
-}
-async function ingestChunks(chunks, options) {
-  console.log(`Total chunks: ${chunks.length}
-`);
-  let changedChunks;
-  let skippedChunks;
-  if (options.force) {
-    changedChunks = chunks;
-    skippedChunks = 0;
-  } else {
-    console.log("Checking for changes...");
-    const chunkIds = chunks.map((c) => c.chunk_id);
-    const existingHashes = await getExistingHashes(chunkIds);
-    changedChunks = chunks.filter((chunk) => {
-      const existingHash = existingHashes.get(chunk.chunk_id);
-      return existingHash !== chunk.content_hash;
-    });
-    skippedChunks = chunks.length - changedChunks.length;
-  }
-  console.log(`Changed chunks: ${changedChunks.length}`);
-  console.log(`Skipped (unchanged): ${skippedChunks}
-`);
-  if (changedChunks.length === 0) {
-    console.log("No changes detected. Done!");
-    return {
-      totalChunks: chunks.length,
-      changedChunks: 0,
-      skippedChunks
-    };
-  }
-  console.log("Upserting to Qdrant...");
-  let processed = 0;
-  for (let i = 0; i < changedChunks.length; i += BATCH_SIZE) {
-    const batch = changedChunks.slice(i, i + BATCH_SIZE);
-    const points = batch.map((chunk) => ({
-      id: chunk.chunk_id,
-      text: chunk.text,
-      payload: chunk
-    }));
-    await withRetry(() => upsertPoints(points));
-    processed += batch.length;
-    console.log(`  Progress: ${processed}/${changedChunks.length}`);
-    await setTimeout(100);
-  }
-  return {
-    totalChunks: chunks.length,
-    changedChunks: changedChunks.length,
-    skippedChunks
-  };
-}
-async function pruneOrphans(currentChunkIds) {
-  const existingIds = await listAllChunkIds();
-  const currentSet = new Set(currentChunkIds);
-  const orphanIds = existingIds.filter((id) => !currentSet.has(id));
-  console.log(`Existing Qdrant chunks: ${existingIds.length}`);
-  console.log(`Orphan chunks to delete: ${orphanIds.length}`);
-  if (orphanIds.length === 0) {
-    return { deletedCount: 0 };
-  }
-  let deleted = 0;
-  for (let i = 0; i < orphanIds.length; i += BATCH_SIZE) {
-    const batch = orphanIds.slice(i, i + BATCH_SIZE);
-    await withRetry(() => deletePointsByChunkIds(batch));
-    deleted += batch.length;
-    console.log(`  Prune progress: ${deleted}/${orphanIds.length}`);
-    await setTimeout(100);
-  }
-  return { deletedCount: deleted };
-}
-
-// src/rag/ingest.ts
-import { readFileSync } from "node:fs";
-
-// src/rag/ingest-config.ts
-var INGEST_CONFIG = {
-  paths: {
-    notes: "markdown/note/**/*.md",
-    books: "markdown/book/**/*.md",
-    articles: "json/article/**/*.json"
-  }
-};
-
 // src/rag/ingest.ts
-async function listFiles() {
-  const files = [];
-  const articleFiles = await globPaths(INGEST_CONFIG.paths.articles);
-  for (const path of articleFiles) {
-    files.push({ path, type: "json", contentType: "articles" });
-  }
-  const noteFiles = await globPaths(INGEST_CONFIG.paths.notes);
-  for (const path of noteFiles) {
-    files.push({ path, type: "markdown", contentType: "notes" });
-  }
-  const bookFiles = await globPaths(INGEST_CONFIG.paths.books);
-  for (const path of bookFiles) {
-    files.push({ path, type: "markdown", contentType: "books" });
-  }
-  return files;
-}
-function parseFile(file) {
-  const content = readFileSync(file.path, "utf-8");
-  if (file.type === "json") {
-    return parseJsonArticle(file.path, content);
-  }
-  return parseMarkdown(file.path, content, file.contentType);
-}
-async function ingest(flags) {
-  console.log("Starting ingest...\n");
-  if (flags.dryRun) {
-    console.log("Dry-run mode: no writes to Qdrant\n");
-  } else {
-    await ensureCollection();
-    const initialStats = await getCollectionStats();
-    console.log(`Initial points count: ${initialStats.pointsCount}
-`);
-  }
-  if (flags.force) {
-    console.log("Force mode enabled: skipping change detection\n");
-  }
-  const files = await listFiles();
-  const articleCount = files.filter((f) => f.contentType === "articles").length;
-  const noteCount = files.filter((f) => f.contentType === "notes").length;
-  const bookCount = files.filter((f) => f.contentType === "books").length;
-  console.log(`Found ${files.length} files to process`);
-  console.log(`  - Articles: ${articleCount}`);
-  console.log(`  - Notes: ${noteCount}`);
-  console.log(`  - Books: ${bookCount}
-`);
-  console.log("Parsing files...");
-  const allChunks = [];
-  for (const file of files) {
-    try {
-      const chunks = parseFile(file);
-      allChunks.push(...chunks);
-    } catch (error) {
-      console.error(`Error parsing ${file.path}:`, error);
-    }
-  }
-  console.log(`Total chunks parsed: ${allChunks.length}`);
-  if (flags.dryRun) {
-    console.log("Dry-run complete. Skipping upsert and prune.");
-    return;
-  }
-  const result = await ingestChunks(allChunks, { force: flags.force });
-  const pruneResult = await pruneOrphans(allChunks.map((c) => c.chunk_id));
-  const finalStats = await getCollectionStats();
-  console.log(`
-Final points count: ${finalStats.pointsCount}`);
-  console.log(
-    `Ingest completed! (${result.changedChunks} changed, ${result.skippedChunks} skipped, ${pruneResult.deletedCount} pruned)`
-  );
-}
-async function main() {
-  if (!process.env.QDRANT_URL) {
-    console.error("QDRANT_URL environment variable is required.");
-    process.exit(1);
-  }
-  const flags = {
-    force: process.argv.includes("--force"),
-    dryRun: process.argv.includes("--dry-run")
-  };
-  try {
-    await ingest(flags);
-  } catch (error) {
-    console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
-    process.exit(2);
-  }
-}
-main().catch((error) => {
-  console.error(error);
-  process.exit(2);
+await runCli("rag-ingest", async () => {
+  const { main } = await import("../chunk-MJYGMGH5.js");
+  await main();
 });