@s-hirano-ist/s-scripts 4.1.0 → 4.3.0

package/dist/chunk-MJYGMGH5.js

@@ -0,0 +1,382 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  RAG_CONFIG,
+  deletePointsByChunkIds,
+  ensureCollection,
+  getCollectionStats,
+  getExistingHashes,
+  listAllChunkIds,
+  upsertPoints
+} from "./chunk-J4VLH4VA.js";
+import {
+  parseMarkdownFrontmatter
+} from "./chunk-HDYQV2Y2.js";
+import {
+  globPaths
+} from "./chunk-AJMTRCPJ.js";
+import {
+  reportCliError
+} from "./chunk-4CXH2QIX.js";
+import "./chunk-JZOR5GA2.js";
+
+// ../search/dist/chunker.js
+import { createHash } from "node:crypto";
+function generateHash(content) {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function generateChunkId(docId, index) {
+  return `${docId}#${index}`;
+}
+function parseJsonArticle(filePath, content) {
+  const json = JSON.parse(content);
+  const docId = `file:${filePath}`;
+  const chunks = [];
+  for (let i = 0; i < json.body.length; i++) {
+    const item = json.body[i];
+    const textParts = [];
+    if (item.title)
+      textParts.push(item.title);
+    if (item.ogTitle && item.ogTitle !== item.title)
+      textParts.push(item.ogTitle);
+    if (item.ogDescription)
+      textParts.push(item.ogDescription);
+    if (item.quote)
+      textParts.push(item.quote);
+    const text = textParts.join("\n");
+    if (!text.trim())
+      continue;
+    const chunkId = generateChunkId(docId, i);
+    chunks.push({
+      type: "bookmark_json",
+      content_type: "articles",
+      top_heading: json.heading,
+      doc_id: docId,
+      chunk_id: chunkId,
+      title: item.title || item.ogTitle || "Untitled",
+      url: item.url,
+      heading_path: [json.heading],
+      text,
+      content_hash: generateHash(text)
+    });
+  }
+  return chunks;
+}
+function parseFrontmatter(content) {
+  const parsed = parseMarkdownFrontmatter(content);
+  const frontmatter = {
+    heading: typeof parsed.data.heading === "string" ? parsed.data.heading : "unknown"
+  };
+  if (typeof parsed.data.description === "string") {
+    frontmatter.description = parsed.data.description;
+  }
+  if (typeof parsed.data.draft === "boolean") {
+    frontmatter.draft = parsed.data.draft;
+  }
+  return { frontmatter, body: parsed.content };
+}
+function pushPreambleSection(state) {
+  if (state.preambleContent.trim()) {
+    state.sections.push({
+      headingPath: [],
+      title: "",
+      content: state.preambleContent,
+      level: 0
+    });
+  }
+}
+function handleHeadingLine(state, headingMatch) {
+  if (!state.preambleDone) {
+    state.preambleDone = true;
+    pushPreambleSection(state);
+  }
+  if (state.currentSection?.content.trim()) {
+    state.sections.push(state.currentSection);
+  }
+  const level = headingMatch[1].length;
+  const title = headingMatch[2];
+  const headingStack = state.headingStack;
+  let top = headingStack.at(-1);
+  while (top && top.level >= level) {
+    headingStack.pop();
+    top = headingStack.at(-1);
+  }
+  headingStack.push({ level, title });
+  state.currentSection = {
+    headingPath: headingStack.map((h) => h.title),
+    title,
+    content: "",
+    level
+  };
+}
+function splitMarkdownByHeadings(content) {
+  const state = {
+    sections: [],
+    currentSection: null,
+    headingStack: [],
+    // Preamble buffer (content before first H2/H3)
+    preambleContent: "",
+    preambleDone: false
+  };
+  for (const line of content.split("\n")) {
+    if (/^#\s+\S/u.test(line)) {
+      continue;
+    }
+    const headingMatch = /^(#{2,3})\s+(\S.*)$/u.exec(line);
+    if (headingMatch) {
+      handleHeadingLine(state, headingMatch);
+    } else if (!state.preambleDone) {
+      state.preambleContent += `${line}
+`;
+    } else if (state.currentSection) {
+      state.currentSection.content += `${line}
+`;
+    }
+  }
+  if (!state.preambleDone) {
+    pushPreambleSection(state);
+  }
+  if (state.currentSection?.content.trim()) {
+    state.sections.push(state.currentSection);
+  }
+  return state.sections;
+}
+function splitByParagraphs(text, maxLength) {
+  if (text.length <= maxLength) {
+    return [text];
+  }
+  const paragraphs = text.split(/\n{2,}/u);
+  const chunks = [];
+  let currentChunk = "";
+  for (const para of paragraphs) {
+    if (currentChunk.length + para.length > maxLength && currentChunk) {
+      chunks.push(currentChunk.trim());
+      currentChunk = para;
+    } else {
+      currentChunk += (currentChunk ? "\n\n" : "") + para;
+    }
+  }
+  if (currentChunk.trim()) {
+    chunks.push(currentChunk.trim());
+  }
+  return chunks;
+}
+function parseMarkdownWithDocId(docId, content, contentType = "notes") {
+  const { frontmatter, body } = parseFrontmatter(content);
+  const chunks = [];
+  if (frontmatter.draft) {
+    return [];
+  }
+  const sections = splitMarkdownByHeadings(body);
+  let chunkIndex = 0;
+  for (const section of sections) {
+    const textChunks = splitByParagraphs(section.content, RAG_CONFIG.chunking.maxChunkLength);
+    for (const text of textChunks) {
+      if (!text.trim())
+        continue;
+      const isPreamble = section.level === 0 && section.title === "";
+      const sectionTitle = isPreamble ? frontmatter.description ?? frontmatter.heading : section.title;
+      const fullHeadingPath = isPreamble ? [frontmatter.heading] : [frontmatter.heading, ...section.headingPath];
+      const chunkId = generateChunkId(docId, chunkIndex);
+      chunks.push({
+        type: "markdown_note",
+        content_type: contentType,
+        top_heading: frontmatter.heading,
+        doc_id: docId,
+        chunk_id: chunkId,
+        title: sectionTitle,
+        heading_path: fullHeadingPath,
+        text,
+        content_hash: generateHash(text)
+      });
+      chunkIndex++;
+    }
+  }
+  return chunks;
+}
+function parseMarkdown(filePath, content, contentType = "notes") {
+  return parseMarkdownWithDocId(`file:${filePath}`, content, contentType);
+}
+
+// ../search/dist/ingest.js
+import { setTimeout } from "node:timers/promises";
+var BATCH_SIZE = 20;
+var MAX_RETRIES = 3;
+var RETRY_DELAY_MS = 2e3;
+async function withRetry(fn, retries = MAX_RETRIES) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (i === retries - 1)
+        throw error;
+      console.log(`  Retry ${i + 1}/${retries} after error...`);
+      await setTimeout(RETRY_DELAY_MS);
+    }
+  }
+  throw new Error("Unreachable");
+}
+async function ingestChunks(chunks, options) {
+  console.log(`Total chunks: ${chunks.length}
+`);
+  let changedChunks;
+  let skippedChunks;
+  if (options.force) {
+    changedChunks = chunks;
+    skippedChunks = 0;
+  } else {
+    console.log("Checking for changes...");
+    const chunkIds = chunks.map((c) => c.chunk_id);
+    const existingHashes = await getExistingHashes(chunkIds);
+    changedChunks = chunks.filter((chunk) => {
+      const existingHash = existingHashes.get(chunk.chunk_id);
+      return existingHash !== chunk.content_hash;
+    });
+    skippedChunks = chunks.length - changedChunks.length;
+  }
+  console.log(`Changed chunks: ${changedChunks.length}`);
+  console.log(`Skipped (unchanged): ${skippedChunks}
+`);
+  if (changedChunks.length === 0) {
+    console.log("No changes detected. Done!");
+    return {
+      totalChunks: chunks.length,
+      changedChunks: 0,
+      skippedChunks
+    };
+  }
+  console.log("Upserting to Qdrant...");
+  let processed = 0;
+  for (let i = 0; i < changedChunks.length; i += BATCH_SIZE) {
+    const batch = changedChunks.slice(i, i + BATCH_SIZE);
+    const points = batch.map((chunk) => ({
+      id: chunk.chunk_id,
+      text: chunk.text,
+      payload: chunk
+    }));
+    await withRetry(() => upsertPoints(points));
+    processed += batch.length;
+    console.log(`  Progress: ${processed}/${changedChunks.length}`);
+    await setTimeout(100);
+  }
+  return {
+    totalChunks: chunks.length,
+    changedChunks: changedChunks.length,
+    skippedChunks
+  };
+}
+async function pruneOrphans(currentChunkIds) {
+  const existingIds = await listAllChunkIds();
+  const currentSet = new Set(currentChunkIds);
+  const orphanIds = existingIds.filter((id) => !currentSet.has(id));
+  console.log(`Existing Qdrant chunks: ${existingIds.length}`);
+  console.log(`Orphan chunks to delete: ${orphanIds.length}`);
+  if (orphanIds.length === 0) {
+    return { deletedCount: 0 };
+  }
+  let deleted = 0;
+  for (let i = 0; i < orphanIds.length; i += BATCH_SIZE) {
+    const batch = orphanIds.slice(i, i + BATCH_SIZE);
+    await withRetry(() => deletePointsByChunkIds(batch));
+    deleted += batch.length;
+    console.log(`  Prune progress: ${deleted}/${orphanIds.length}`);
+    await setTimeout(100);
+  }
+  return { deletedCount: deleted };
+}
+
+// src/rag/ingest-command.ts
+import { readFileSync } from "node:fs";
+
+// src/rag/ingest-config.ts
+var INGEST_CONFIG = {
+  paths: {
+    notes: "markdown/note/**/*.md",
+    books: "markdown/book/**/*.md",
+    articles: "json/article/**/*.json"
+  }
+};
+
+// src/rag/ingest-command.ts
+async function listFiles() {
+  const files = [];
+  const articleFiles = await globPaths(INGEST_CONFIG.paths.articles);
+  for (const path of articleFiles) {
+    files.push({ path, type: "json", contentType: "articles" });
+  }
+  const noteFiles = await globPaths(INGEST_CONFIG.paths.notes);
+  for (const path of noteFiles) {
+    files.push({ path, type: "markdown", contentType: "notes" });
+  }
+  const bookFiles = await globPaths(INGEST_CONFIG.paths.books);
+  for (const path of bookFiles) {
+    files.push({ path, type: "markdown", contentType: "books" });
+  }
+  return files;
+}
+function parseFile(file) {
+  const content = readFileSync(file.path, "utf-8");
+  if (file.type === "json") {
+    return parseJsonArticle(file.path, content);
+  }
+  return parseMarkdown(file.path, content, file.contentType);
+}
+async function ingest(flags) {
+  console.log("Starting ingest...\n");
+  if (flags.dryRun) {
+    console.log("Dry-run mode: no writes to Qdrant\n");
+  } else {
+    await ensureCollection();
+    const initialStats = await getCollectionStats();
+    console.log(`Initial points count: ${initialStats.pointsCount}
+`);
+  }
+  if (flags.force) {
+    console.log("Force mode enabled: skipping change detection\n");
+  }
+  const files = await listFiles();
+  const articleCount = files.filter((f) => f.contentType === "articles").length;
+  const noteCount = files.filter((f) => f.contentType === "notes").length;
+  const bookCount = files.filter((f) => f.contentType === "books").length;
+  console.log(`Found ${files.length} files to process`);
+  console.log(`  - Articles: ${articleCount}`);
+  console.log(`  - Notes: ${noteCount}`);
+  console.log(`  - Books: ${bookCount}
+`);
+  console.log("Parsing files...");
+  const allChunks = [];
+  for (const file of files) {
+    try {
+      const chunks = parseFile(file);
+      allChunks.push(...chunks);
+    } catch (error) {
+      reportCliError(error, "rag-ingest");
+      console.error(`Error parsing ${file.path}:`, error);
+    }
+  }
+  console.log(`Total chunks parsed: ${allChunks.length}`);
+  if (flags.dryRun) {
+    console.log("Dry-run complete. Skipping upsert and prune.");
+    return;
+  }
+  const result = await ingestChunks(allChunks, { force: flags.force });
+  const pruneResult = await pruneOrphans(allChunks.map((c) => c.chunk_id));
+  const finalStats = await getCollectionStats();
+  console.log(`
+Final points count: ${finalStats.pointsCount}`);
+  console.log(
+    `Ingest completed! (${result.changedChunks} changed, ${result.skippedChunks} skipped, ${pruneResult.deletedCount} pruned)`
+  );
+}
+async function main() {
+  if (!process.env.QDRANT_URL) {
+    throw new Error("QDRANT_URL environment variable is required.");
+  }
+  const flags = {
+    force: process.argv.includes("--force"),
+    dryRun: process.argv.includes("--dry-run")
+  };
+  await ingest(flags);
+}
+export {
+  main
+};

package/dist/chunk-MVYAZJB2.js

@@ -0,0 +1,160 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../storage/dist/src/client.js
+import { S3Client, CreateBucketCommand, DeleteBucketCommand, PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand, paginateListObjectsV2 } from "@aws-sdk/client-s3";
+import { createWriteStream } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+function loadStorageConfig(env = process.env) {
+  const required = (key) => {
+    const value = env[key];
+    if (!value)
+      throw new Error(`${key} is required`);
+    return value;
+  };
+  const endpoint = required("S3_ENDPOINT");
+  const url = new URL(endpoint);
+  if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.pathname !== "/")
+    throw new Error("S3_ENDPOINT must be an HTTP(S) origin");
+  const pathStyle = env.S3_FORCE_PATH_STYLE ?? "false";
+  if (pathStyle !== "true" && pathStyle !== "false")
+    throw new Error("S3_FORCE_PATH_STYLE must be true or false");
+  return {
+    endpoint,
+    region: required("S3_REGION"),
+    accessKeyId: required("S3_ACCESS_KEY_ID"),
+    secretAccessKey: required("S3_SECRET_ACCESS_KEY"),
+    forcePathStyle: pathStyle === "true"
+  };
+}
+function isObjectNotFound(error) {
+  if (typeof error !== "object" || error === null)
+    return false;
+  const value = error;
+  return value.$metadata?.httpStatusCode === 404 && (value.name === "NotFound" || value.name === "NoSuchKey");
+}
+function createObjectStorageClient(config = loadStorageConfig()) {
+  const client = new S3Client({
+    endpoint: config.endpoint,
+    region: config.region,
+    credentials: {
+      accessKeyId: config.accessKeyId,
+      secretAccessKey: config.secretAccessKey
+    },
+    forcePathStyle: config.forcePathStyle
+  });
+  return {
+    async createBucket(bucket) {
+      await client.send(new CreateBucketCommand({ Bucket: bucket }));
+    },
+    async deleteBucket(bucket) {
+      await client.send(new DeleteBucketCommand({ Bucket: bucket }));
+    },
+    async putObject(bucket, key, body) {
+      const result = await client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));
+      return { etag: result.ETag, versionId: result.VersionId };
+    },
+    async getObject(bucket, key) {
+      const result = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+      if (!(result.Body instanceof Readable))
+        throw new TypeError("Object storage returned no Node readable body");
+      return result.Body;
+    },
+    async downloadObject(bucket, key, destination) {
+      const result = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+      if (!(result.Body instanceof Readable))
+        throw new TypeError("Object storage returned no Node readable body");
+      await pipeline(result.Body, createWriteStream(destination));
+    },
+    async statObject(bucket, key) {
+      const result = await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
+      return {
+        size: result.ContentLength,
+        etag: result.ETag,
+        lastModified: result.LastModified,
+        metaData: result.Metadata ?? {}
+      };
+    },
+    async removeObject(bucket, key) {
+      await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }));
+    },
+    async listObjects(bucket, prefix) {
+      const keys = [];
+      for await (const page of paginateListObjectsV2({ client }, { Bucket: bucket, Prefix: prefix })) {
+        for (const object of page.Contents ?? [])
+          if (object.Key)
+            keys.push(object.Key);
+      }
+      return keys;
+    },
+    destroy() {
+      client.destroy();
+    }
+  };
+}
+
+// ../storage/dist/index.js
+import { S3ServiceException } from "@aws-sdk/client-s3";
+
+// ../storage/dist/src/errors.js
+var StorageOperationError = class extends Error {
+  context;
+  constructor(context, cause) {
+    super(`Storage ${context.operation} failed for ${context.objectKey}`, {
+      cause
+    });
+    this.name = "StorageOperationError";
+    this.context = context;
+  }
+};
+
+// ../storage/dist/src/storage-service.js
+function createStorageService(client, bucketName, pathConfig) {
+  function buildObjectKey(path, isThumbnail) {
+    return `${isThumbnail ? pathConfig.thumbnailPrefix : pathConfig.originalPrefix}/${path}`;
+  }
+  function wrapStorageError(operation, objectKey, isThumbnail, error) {
+    throw new StorageOperationError({ operation, objectKey, bucketName, isThumbnail }, error);
+  }
+  return {
+    async uploadImage(path, bytes, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.putObject(bucketName, objKey, Buffer.from(bytes));
+      } catch (error) {
+        wrapStorageError("uploadImage", objKey, isThumbnail, error);
+      }
+    },
+    async getImage(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        return await client.getObject(bucketName, objKey);
+      } catch (error) {
+        return wrapStorageError("getImage", objKey, isThumbnail, error);
+      }
+    },
+    async getImageOrThrow(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.statObject(bucketName, objKey);
+      } catch (error) {
+        wrapStorageError("getImageOrThrow", objKey, isThumbnail, error);
+      }
+    },
+    async deleteImage(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.removeObject(bucketName, objKey);
+      } catch (error) {
+        wrapStorageError("deleteImage", objKey, isThumbnail, error);
+      }
+    }
+  };
+}
+
+export {
+  isObjectNotFound,
+  createObjectStorageClient,
+  StorageOperationError,
+  createStorageService
+};