@s-hirano-ist/s-scripts 1.26.7 → 4.0.0

package/dist/rag/ingest.js

@@ -1,107 +1,386 @@
 #!/usr/bin/env node
-import { parseJsonArticle, parseMarkdown, } from "@s-hirano-ist/s-search/chunker";
-import { ingestChunks, pruneOrphans } from "@s-hirano-ist/s-search/ingest";
-import { ensureCollection, getCollectionStats, } from "@s-hirano-ist/s-search/qdrant-client";
-import { readFileSync } from "node:fs";
-import { globPaths } from "../lib/glob-paths.js";
-import { INGEST_CONFIG } from "./ingest-config.js";
-/**
- * List all files to process with content type information
- */
-async function listFiles() {
-    const files = [];
-    // Articles (JSON)
-    const articleFiles = await globPaths(INGEST_CONFIG.paths.articles);
-    for (const path of articleFiles) {
-        files.push({ path, type: "json", contentType: "articles" });
-    }
-    // Notes (Markdown)
-    const noteFiles = await globPaths(INGEST_CONFIG.paths.notes);
-    for (const path of noteFiles) {
-        files.push({ path, type: "markdown", contentType: "notes" });
-    }
-    // Books (Markdown)
-    const bookFiles = await globPaths(INGEST_CONFIG.paths.books);
-    for (const path of bookFiles) {
-        files.push({ path, type: "markdown", contentType: "books" });
-    }
-    return files;
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  parseMarkdownFrontmatter
+} from "../chunk-HDYQV2Y2.js";
+import {
+  globPaths
+} from "../chunk-AJMTRCPJ.js";
+import {
+  RAG_CONFIG,
+  deletePointsByChunkIds,
+  ensureCollection,
+  getCollectionStats,
+  getExistingHashes,
+  listAllChunkIds,
+  upsertPoints
+} from "../chunk-J4VLH4VA.js";
+import "../chunk-JZOR5GA2.js";
+
+// ../search/dist/chunker.js
+import { createHash } from "node:crypto";
+function generateHash(content) {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
 }
-/**
- * Parse a single file into chunks
- */
-function parseFile(file) {
-    const content = readFileSync(file.path, "utf-8");
-    if (file.type === "json") {
-        return parseJsonArticle(file.path, content);
-    }
-    return parseMarkdown(file.path, content, file.contentType);
+function generateChunkId(docId, index) {
+  return `${docId}#${index}`;
 }
-/**
- * CLI entry point: list files, parse into chunks, delegate to ingestChunks
- */
-async function ingest(flags) {
-    console.log("Starting ingest...\n");
-    if (flags.dryRun) {
-        console.log("Dry-run mode: no writes to Qdrant\n");
-    }
-    else {
-        await ensureCollection();
-        const initialStats = await getCollectionStats();
-        console.log(`Initial points count: ${initialStats.pointsCount}\n`);
+function parseJsonArticle(filePath, content) {
+  const json = JSON.parse(content);
+  const docId = `file:${filePath}`;
+  const chunks = [];
+  for (let i = 0; i < json.body.length; i++) {
+    const item = json.body[i];
+    const textParts = [];
+    if (item.title)
+      textParts.push(item.title);
+    if (item.ogTitle && item.ogTitle !== item.title)
+      textParts.push(item.ogTitle);
+    if (item.ogDescription)
+      textParts.push(item.ogDescription);
+    if (item.quote)
+      textParts.push(item.quote);
+    const text = textParts.join("\n");
+    if (!text.trim())
+      continue;
+    const chunkId = generateChunkId(docId, i);
+    chunks.push({
+      type: "bookmark_json",
+      content_type: "articles",
+      top_heading: json.heading,
+      doc_id: docId,
+      chunk_id: chunkId,
+      title: item.title || item.ogTitle || "Untitled",
+      url: item.url,
+      heading_path: [json.heading],
+      text,
+      content_hash: generateHash(text)
+    });
+  }
+  return chunks;
+}
+function parseFrontmatter(content) {
+  const parsed = parseMarkdownFrontmatter(content);
+  const frontmatter = {
+    heading: typeof parsed.data.heading === "string" ? parsed.data.heading : "unknown"
+  };
+  if (typeof parsed.data.description === "string") {
+    frontmatter.description = parsed.data.description;
+  }
+  if (typeof parsed.data.draft === "boolean") {
+    frontmatter.draft = parsed.data.draft;
+  }
+  return { frontmatter, body: parsed.content };
+}
+function pushPreambleSection(state) {
+  if (state.preambleContent.trim()) {
+    state.sections.push({
+      headingPath: [],
+      title: "",
+      content: state.preambleContent,
+      level: 0
+    });
+  }
+}
+function handleHeadingLine(state, headingMatch) {
+  if (!state.preambleDone) {
+    state.preambleDone = true;
+    pushPreambleSection(state);
+  }
+  if (state.currentSection?.content.trim()) {
+    state.sections.push(state.currentSection);
+  }
+  const level = headingMatch[1].length;
+  const title = headingMatch[2];
+  const headingStack = state.headingStack;
+  let top = headingStack.at(-1);
+  while (top && top.level >= level) {
+    headingStack.pop();
+    top = headingStack.at(-1);
+  }
+  headingStack.push({ level, title });
+  state.currentSection = {
+    headingPath: headingStack.map((h) => h.title),
+    title,
+    content: "",
+    level
+  };
+}
+function splitMarkdownByHeadings(content) {
+  const state = {
+    sections: [],
+    currentSection: null,
+    headingStack: [],
+    // Preamble buffer (content before first H2/H3)
+    preambleContent: "",
+    preambleDone: false
+  };
+  for (const line of content.split("\n")) {
+    if (/^#\s+\S/u.test(line)) {
+      continue;
     }
-    if (flags.force) {
-        console.log("Force mode enabled: skipping change detection\n");
+    const headingMatch = /^(#{2,3})\s+(\S.*)$/u.exec(line);
+    if (headingMatch) {
+      handleHeadingLine(state, headingMatch);
+    } else if (!state.preambleDone) {
+      state.preambleContent += `${line}
+`;
+    } else if (state.currentSection) {
+      state.currentSection.content += `${line}
+`;
     }
-    const files = await listFiles();
-    const articleCount = files.filter((f) => f.contentType === "articles").length;
-    const noteCount = files.filter((f) => f.contentType === "notes").length;
-    const bookCount = files.filter((f) => f.contentType === "books").length;
-    console.log(`Found ${files.length} files to process`);
-    console.log(`  - Articles: ${articleCount}`);
-    console.log(`  - Notes: ${noteCount}`);
-    console.log(`  - Books: ${bookCount}\n`);
-    console.log("Parsing files...");
-    const allChunks = [];
-    for (const file of files) {
-        try {
-            const chunks = parseFile(file);
-            allChunks.push(...chunks);
-        }
-        catch (error) {
-            console.error(`Error parsing ${file.path}:`, error);
-        }
+  }
+  if (!state.preambleDone) {
+    pushPreambleSection(state);
+  }
+  if (state.currentSection?.content.trim()) {
+    state.sections.push(state.currentSection);
+  }
+  return state.sections;
+}
+function splitByParagraphs(text, maxLength) {
+  if (text.length <= maxLength) {
+    return [text];
+  }
+  const paragraphs = text.split(/\n{2,}/u);
+  const chunks = [];
+  let currentChunk = "";
+  for (const para of paragraphs) {
+    if (currentChunk.length + para.length > maxLength && currentChunk) {
+      chunks.push(currentChunk.trim());
+      currentChunk = para;
+    } else {
+      currentChunk += (currentChunk ? "\n\n" : "") + para;
     }
-    console.log(`Total chunks parsed: ${allChunks.length}`);
-    if (flags.dryRun) {
-        console.log("Dry-run complete. Skipping upsert and prune.");
-        return;
+  }
+  if (currentChunk.trim()) {
+    chunks.push(currentChunk.trim());
+  }
+  return chunks;
+}
+function parseMarkdownWithDocId(docId, content, contentType = "notes") {
+  const { frontmatter, body } = parseFrontmatter(content);
+  const chunks = [];
+  if (frontmatter.draft) {
+    return [];
+  }
+  const sections = splitMarkdownByHeadings(body);
+  let chunkIndex = 0;
+  for (const section of sections) {
+    const textChunks = splitByParagraphs(section.content, RAG_CONFIG.chunking.maxChunkLength);
+    for (const text of textChunks) {
+      if (!text.trim())
+        continue;
+      const isPreamble = section.level === 0 && section.title === "";
+      const sectionTitle = isPreamble ? frontmatter.description ?? frontmatter.heading : section.title;
+      const fullHeadingPath = isPreamble ? [frontmatter.heading] : [frontmatter.heading, ...section.headingPath];
+      const chunkId = generateChunkId(docId, chunkIndex);
+      chunks.push({
+        type: "markdown_note",
+        content_type: contentType,
+        top_heading: frontmatter.heading,
+        doc_id: docId,
+        chunk_id: chunkId,
+        title: sectionTitle,
+        heading_path: fullHeadingPath,
+        text,
+        content_hash: generateHash(text)
+      });
+      chunkIndex++;
     }
-    const result = await ingestChunks(allChunks, { force: flags.force });
-    const pruneResult = await pruneOrphans(allChunks.map((c) => c.chunk_id));
-    const finalStats = await getCollectionStats();
-    console.log(`\nFinal points count: ${finalStats.pointsCount}`);
-    console.log(`Ingest completed! (${result.changedChunks} changed, ${result.skippedChunks} skipped, ${pruneResult.deletedCount} pruned)`);
+  }
+  return chunks;
 }
-async function main() {
-    if (!process.env.QDRANT_URL) {
-        console.error("QDRANT_URL environment variable is required.");
-        process.exit(1);
+function parseMarkdown(filePath, content, contentType = "notes") {
+  return parseMarkdownWithDocId(`file:${filePath}`, content, contentType);
+}
+
+// ../search/dist/ingest.js
+import { setTimeout } from "node:timers/promises";
+var BATCH_SIZE = 20;
+var MAX_RETRIES = 3;
+var RETRY_DELAY_MS = 2e3;
+async function withRetry(fn, retries = MAX_RETRIES) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (i === retries - 1)
+        throw error;
+      console.log(`  Retry ${i + 1}/${retries} after error...`);
+      await setTimeout(RETRY_DELAY_MS);
     }
-    const flags = {
-        force: process.argv.includes("--force"),
-        dryRun: process.argv.includes("--dry-run"),
+  }
+  throw new Error("Unreachable");
+}
+async function ingestChunks(chunks, options) {
+  console.log(`Total chunks: ${chunks.length}
+`);
+  let changedChunks;
+  let skippedChunks;
+  if (options.force) {
+    changedChunks = chunks;
+    skippedChunks = 0;
+  } else {
+    console.log("Checking for changes...");
+    const chunkIds = chunks.map((c) => c.chunk_id);
+    const existingHashes = await getExistingHashes(chunkIds);
+    changedChunks = chunks.filter((chunk) => {
+      const existingHash = existingHashes.get(chunk.chunk_id);
+      return existingHash !== chunk.content_hash;
+    });
+    skippedChunks = chunks.length - changedChunks.length;
+  }
+  console.log(`Changed chunks: ${changedChunks.length}`);
+  console.log(`Skipped (unchanged): ${skippedChunks}
+`);
+  if (changedChunks.length === 0) {
+    console.log("No changes detected. Done!");
+    return {
+      totalChunks: chunks.length,
+      changedChunks: 0,
+      skippedChunks
     };
+  }
+  console.log("Upserting to Qdrant...");
+  let processed = 0;
+  for (let i = 0; i < changedChunks.length; i += BATCH_SIZE) {
+    const batch = changedChunks.slice(i, i + BATCH_SIZE);
+    const points = batch.map((chunk) => ({
+      id: chunk.chunk_id,
+      text: chunk.text,
+      payload: chunk
+    }));
+    await withRetry(() => upsertPoints(points));
+    processed += batch.length;
+    console.log(`  Progress: ${processed}/${changedChunks.length}`);
+    await setTimeout(100);
+  }
+  return {
+    totalChunks: chunks.length,
+    changedChunks: changedChunks.length,
+    skippedChunks
+  };
+}
+async function pruneOrphans(currentChunkIds) {
+  const existingIds = await listAllChunkIds();
+  const currentSet = new Set(currentChunkIds);
+  const orphanIds = existingIds.filter((id) => !currentSet.has(id));
+  console.log(`Existing Qdrant chunks: ${existingIds.length}`);
+  console.log(`Orphan chunks to delete: ${orphanIds.length}`);
+  if (orphanIds.length === 0) {
+    return { deletedCount: 0 };
+  }
+  let deleted = 0;
+  for (let i = 0; i < orphanIds.length; i += BATCH_SIZE) {
+    const batch = orphanIds.slice(i, i + BATCH_SIZE);
+    await withRetry(() => deletePointsByChunkIds(batch));
+    deleted += batch.length;
+    console.log(`  Prune progress: ${deleted}/${orphanIds.length}`);
+    await setTimeout(100);
+  }
+  return { deletedCount: deleted };
+}
+
+// src/rag/ingest.ts
+import { readFileSync } from "node:fs";
+
+// src/rag/ingest-config.ts
+var INGEST_CONFIG = {
+  paths: {
+    notes: "markdown/note/**/*.md",
+    books: "markdown/book/**/*.md",
+    articles: "json/article/**/*.json"
+  }
+};
+
+// src/rag/ingest.ts
+async function listFiles() {
+  const files = [];
+  const articleFiles = await globPaths(INGEST_CONFIG.paths.articles);
+  for (const path of articleFiles) {
+    files.push({ path, type: "json", contentType: "articles" });
+  }
+  const noteFiles = await globPaths(INGEST_CONFIG.paths.notes);
+  for (const path of noteFiles) {
+    files.push({ path, type: "markdown", contentType: "notes" });
+  }
+  const bookFiles = await globPaths(INGEST_CONFIG.paths.books);
+  for (const path of bookFiles) {
+    files.push({ path, type: "markdown", contentType: "books" });
+  }
+  return files;
+}
+function parseFile(file) {
+  const content = readFileSync(file.path, "utf-8");
+  if (file.type === "json") {
+    return parseJsonArticle(file.path, content);
+  }
+  return parseMarkdown(file.path, content, file.contentType);
+}
+async function ingest(flags) {
+  console.log("Starting ingest...\n");
+  if (flags.dryRun) {
+    console.log("Dry-run mode: no writes to Qdrant\n");
+  } else {
+    await ensureCollection();
+    const initialStats = await getCollectionStats();
+    console.log(`Initial points count: ${initialStats.pointsCount}
+`);
+  }
+  if (flags.force) {
+    console.log("Force mode enabled: skipping change detection\n");
+  }
+  const files = await listFiles();
+  const articleCount = files.filter((f) => f.contentType === "articles").length;
+  const noteCount = files.filter((f) => f.contentType === "notes").length;
+  const bookCount = files.filter((f) => f.contentType === "books").length;
+  console.log(`Found ${files.length} files to process`);
+  console.log(`  - Articles: ${articleCount}`);
+  console.log(`  - Notes: ${noteCount}`);
+  console.log(`  - Books: ${bookCount}
+`);
+  console.log("Parsing files...");
+  const allChunks = [];
+  for (const file of files) {
     try {
-        await ingest(flags);
-    }
-    catch (error) {
-        console.error("❌ エラーが発生しました:", error);
-        process.exit(2);
+      const chunks = parseFile(file);
+      allChunks.push(...chunks);
+    } catch (error) {
+      console.error(`Error parsing ${file.path}:`, error);
     }
+  }
+  console.log(`Total chunks parsed: ${allChunks.length}`);
+  if (flags.dryRun) {
+    console.log("Dry-run complete. Skipping upsert and prune.");
+    return;
+  }
+  const result = await ingestChunks(allChunks, { force: flags.force });
+  const pruneResult = await pruneOrphans(allChunks.map((c) => c.chunk_id));
+  const finalStats = await getCollectionStats();
+  console.log(`
+Final points count: ${finalStats.pointsCount}`);
+  console.log(
+    `Ingest completed! (${result.changedChunks} changed, ${result.skippedChunks} skipped, ${pruneResult.deletedCount} pruned)`
+  );
 }
-main().catch((error) => {
-    console.error(error);
+async function main() {
+  if (!process.env.QDRANT_URL) {
+    console.error("QDRANT_URL environment variable is required.");
+    process.exit(1);
+  }
+  const flags = {
+    force: process.argv.includes("--force"),
+    dryRun: process.argv.includes("--dry-run")
+  };
+  try {
+    await ingest(flags);
+  } catch (error) {
+    console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
     process.exit(2);
+  }
+}
+main().catch((error) => {
+  console.error(error);
+  process.exit(2);
 });
-//# sourceMappingURL=ingest.js.map

package/dist/rag/search.js

@@ -1,87 +1,91 @@
 #!/usr/bin/env node
-import { ensureCollection, search as qdrantSearch, } from "@s-hirano-ist/s-search/qdrant-client";
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  ensureCollection,
+  search
+} from "../chunk-J4VLH4VA.js";
+import "../chunk-JZOR5GA2.js";
+
+// src/rag/search.ts
 function parseArgs() {
-    const args = process.argv.slice(2);
-    const options = {};
-    const positional = [];
-    for (let i = 0; i < args.length; i++) {
-        const arg = args[i];
-        if (arg === "--top-k" && i + 1 < args.length) {
-            options.topK = Number(args[++i]);
-        }
-        else if (arg === "--type" && i + 1 < args.length) {
-            options.type = args[++i];
-        }
-        else if (arg === "--heading" && i + 1 < args.length) {
-            options.heading = args[++i];
-        }
-        else if (arg === "--content-type" && i + 1 < args.length) {
-            const value = args[++i];
-            const types = value.split(",");
-            options.contentType = types.length === 1 ? types[0] : types;
-        }
-        else {
-            positional.push(arg);
-        }
+  const args = process.argv.slice(2);
+  const options = {};
+  const positional = [];
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "--top-k" && i + 1 < args.length) {
+      options.topK = Number(args[++i]);
+    } else if (arg === "--type" && i + 1 < args.length) {
+      options.type = args[++i];
+    } else if (arg === "--heading" && i + 1 < args.length) {
+      options.heading = args[++i];
+    } else if (arg === "--content-type" && i + 1 < args.length) {
+      const value = args[++i];
+      const types = value.split(",");
+      options.contentType = types.length === 1 ? types[0] : types;
+    } else {
+      positional.push(arg);
     }
-    const query = positional.join(" ");
-    if (!query) {
-        console.error("Usage: rag-search <query> [--top-k N] [--type markdown_note|bookmark_json] [--heading HEADING] [--content-type articles|books|notes]");
-        process.exit(1);
-    }
-    return { query, options };
+  }
+  const query = positional.join(" ");
+  if (!query) {
+    console.error(
+      "Usage: rag-search <query> [--top-k N] [--type markdown_note|bookmark_json] [--heading HEADING] [--content-type articles|books|notes]"
+    );
+    process.exit(1);
+  }
+  return { query, options };
 }
 async function searchContent(query, options = {}) {
-    const results = await qdrantSearch(query, {
-        topK: options.topK,
-        filter: {
-            type: options.type,
-            top_heading: options.heading,
-            content_type: options.contentType,
-        },
-    });
-    return { results, query, totalResults: results.length };
-}
-async function search() {
-    const { query, options } = parseArgs();
-    console.log(`Searching for: "${query}"`);
-    if (options.topK)
-        console.log(`  top-k: ${options.topK}`);
-    if (options.type)
-        console.log(`  type: ${options.type}`);
-    if (options.heading)
-        console.log(`  heading: ${options.heading}`);
-    if (options.contentType)
-        console.log(`  content-type: ${Array.isArray(options.contentType) ? options.contentType.join(",") : options.contentType}`);
-    console.log();
-    await ensureCollection();
-    const response = await searchContent(query, options);
-    console.log(`Found ${response.totalResults} results:\n`);
-    for (const [i, result] of response.results.entries()) {
-        console.log(`--- Result ${i + 1} (score: ${result.score.toFixed(4)}) ---`);
-        console.log(`Title: ${result.title}`);
-        if (result.url)
-            console.log(`URL: ${result.url}`);
-        console.log(`Type: ${result.type}`);
-        console.log(`Content-Type: ${result.content_type}`);
-        console.log(`Heading: ${result.heading_path.join(" > ")}`);
-        console.log(`\n${result.text}\n`);
+  const results = await search(query, {
+    topK: options.topK,
+    filter: {
+      type: options.type,
+      top_heading: options.heading,
+      content_type: options.contentType
     }
+  });
+  return { results, query, totalResults: results.length };
+}
+async function search2() {
+  const { query, options } = parseArgs();
+  console.log(`Searching for: "${query}"`);
+  if (options.topK) console.log(`  top-k: ${options.topK}`);
+  if (options.type) console.log(`  type: ${options.type}`);
+  if (options.heading) console.log(`  heading: ${options.heading}`);
+  if (options.contentType)
+    console.log(
+      `  content-type: ${Array.isArray(options.contentType) ? options.contentType.join(",") : options.contentType}`
+    );
+  console.log();
+  await ensureCollection();
+  const response = await searchContent(query, options);
+  console.log(`Found ${response.totalResults} results:
+`);
+  for (const [i, result] of response.results.entries()) {
+    console.log(`--- Result ${i + 1} (score: ${result.score.toFixed(4)}) ---`);
+    console.log(`Title: ${result.title}`);
+    if (result.url) console.log(`URL: ${result.url}`);
+    console.log(`Type: ${result.type}`);
+    console.log(`Content-Type: ${result.content_type}`);
+    console.log(`Heading: ${result.heading_path.join(" > ")}`);
+    console.log(`
+${result.text}
+`);
+  }
 }
 async function main() {
-    if (!process.env.QDRANT_URL) {
-        throw new Error("QDRANT_URL environment variable is required.");
-    }
-    try {
-        await search();
-    }
-    catch (error) {
-        console.error("❌ エラーが発生しました:", error);
-        process.exit(1);
-    }
+  if (!process.env.QDRANT_URL) {
+    throw new Error("QDRANT_URL environment variable is required.");
+  }
+  try {
+    await search2();
+  } catch (error) {
+    console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
+    process.exit(1);
+  }
 }
 main().catch((error) => {
-    console.error(error);
-    process.exit(1);
+  console.error(error);
+  process.exit(1);
 });
-//# sourceMappingURL=search.js.map