contentbase 0.1.8 → 0.4.0

package/src/search/luca-semantic-search.ts

@@ -0,0 +1,75 @@
+import { existsSync, readdirSync } from 'node:fs'
+import path from 'node:path'
+
+export type EmbeddingProvider = 'local' | 'openai'
+export type ChunkStrategy = 'section' | 'fixed' | 'document'
+
+export interface SemanticSearchConfig {
+  dbPath: string
+  embeddingProvider: EmbeddingProvider
+  embeddingModel: string
+  chunkStrategy: ChunkStrategy
+  chunkSize: number
+  chunkOverlap: number
+}
+
+export interface SemanticSearchOptions extends Partial<SemanticSearchConfig> {}
+
+export const DEFAULT_SEMANTIC_SEARCH_OPTIONS = {
+  embeddingProvider: 'openai' as EmbeddingProvider,
+  embeddingModel: 'text-embedding-3-small',
+  chunkStrategy: 'section' as ChunkStrategy,
+  chunkSize: 900,
+  chunkOverlap: 0.15,
+}
+
+export function getSearchDbPath(rootPath: string): string {
+  return path.join(rootPath, '.contentbase/search.sqlite')
+}
+
+export function hasSearchIndex(rootPath: string): boolean {
+  const dbDir = path.join(rootPath, '.contentbase')
+  if (!existsSync(dbDir)) return false
+
+  try {
+    const files = readdirSync(dbDir) as string[]
+    return files.some((file) => file.startsWith('search.') && file.endsWith('.sqlite'))
+  } catch {
+    return false
+  }
+}
+
+export function buildSemanticSearchConfig(rootPath: string, options: SemanticSearchOptions = {}): SemanticSearchConfig {
+  return {
+    dbPath: options.dbPath ?? getSearchDbPath(rootPath),
+    embeddingProvider: options.embeddingProvider ?? DEFAULT_SEMANTIC_SEARCH_OPTIONS.embeddingProvider,
+    embeddingModel: options.embeddingModel ?? DEFAULT_SEMANTIC_SEARCH_OPTIONS.embeddingModel,
+    chunkStrategy: options.chunkStrategy ?? DEFAULT_SEMANTIC_SEARCH_OPTIONS.chunkStrategy,
+    chunkSize: options.chunkSize ?? DEFAULT_SEMANTIC_SEARCH_OPTIONS.chunkSize,
+    chunkOverlap: options.chunkOverlap ?? DEFAULT_SEMANTIC_SEARCH_OPTIONS.chunkOverlap,
+  }
+}
+
+export async function loadSemanticSearchClass(): Promise<any> {
+  const { SemanticSearch } = await import('luca/agi')
+  return SemanticSearch
+}
+
+export async function ensureSemanticSearchAttached(container: any, SemanticSearchClass?: any): Promise<any> {
+  const SemanticSearch = SemanticSearchClass ?? await loadSemanticSearchClass()
+  if (!container.features.available.includes('semanticSearch')) {
+    ;(SemanticSearch as any).attach(container as any)
+  }
+  return SemanticSearch
+}
+
+export async function createSemanticSearch(container: any, rootPath: string, options: SemanticSearchOptions = {}): Promise<any> {
+  await ensureSemanticSearchAttached(container)
+  return container.feature('semanticSearch', buildSemanticSearchConfig(rootPath, options))
+}
+
+export async function getInitializedSemanticSearch(container: any, rootPath: string, options: SemanticSearchOptions = {}): Promise<any> {
+  const semanticSearch = await createSemanticSearch(container, rootPath, options)
+  await semanticSearch.initDb()
+  return semanticSearch
+}

package/src/types.ts

@@ -15,6 +15,54 @@ export interface CollectionItem {
   size: number;
 }
 
+// ─── Storage adapter ───
+
+/** File metadata returned by stat and listFiles */
+export interface FileStat {
+  createdAt: Date;
+  updatedAt: Date;
+  size: number;
+}
+
+/** A file entry returned from listFiles — key plus pre-fetched stat */
+export interface FileEntry {
+  key: string;
+  stat: FileStat;
+}
+
+/**
+ * Abstracts file system operations so Collections can run on top of
+ * any storage backend (local disk, Cloudflare R2, S3, etc.).
+ *
+ * Implement this interface and pass it as `adapter` in CollectionOptions.
+ */
+export interface StorageAdapter {
+  /** List all file keys under rootPath whose names match the extension pattern.
+   *  Returns key + pre-fetched stat so load() avoids per-file round-trips. */
+  listFiles(rootPath: string, match: RegExp): Promise<FileEntry[]>;
+
+  /** Read file content as a UTF-8 string */
+  readFile(key: string): Promise<string>;
+
+  /** Fetch metadata for a single key */
+  stat(key: string): Promise<FileStat>;
+
+  /** Write content to key, creating any parent directories/prefixes as needed */
+  writeFile(key: string, content: string): Promise<void>;
+
+  /** Delete the file at key (silently succeeds if the key does not exist) */
+  deleteFile(key: string): Promise<void>;
+
+  /** Join a root and path segments into a full key (e.g. path.resolve for local FS) */
+  join(root: string, ...parts: string[]): string;
+
+  /** Strip the root prefix from a full key to get a relative key */
+  relative(root: string, key: string): string;
+
+  /** Return the file extension including the leading dot (e.g. ".md") */
+  extname(key: string): string;
+}
+
 /** Options when constructing a Collection */
 export interface CollectionOptions {
   rootPath: string;
@@ -24,6 +72,12 @@ export interface CollectionOptions {
   autoDiscover?: boolean;
   /** Optional custom module loader for models.ts discovery. When provided, used instead of native import(). Enables VM-based loading in compiled binaries where node_modules aren't available. */
   moduleLoader?: (filePath: string) => Record<string, any> | Promise<Record<string, any>>;
+  /**
+   * Storage adapter for file system operations.
+   * Defaults to NodeStorageAdapter (local disk via fs/promises).
+   * Swap in a custom adapter to use Cloudflare R2, S3, or any other backend.
+   */
+  adapter?: StorageAdapter;
 }
 
 // ─── Section system ───

package/src/utils/index.ts

@@ -10,3 +10,5 @@ export { parseTable } from "./parse-table";
 export { normalizeHeadings } from "./normalize-headings";
 export { readDirectory } from "./read-directory";
 export { matchPattern, matchPatterns } from "./match-pattern";
+export { stripMarkdown } from "./strip-markdown";
+export type { StripMarkdownOptions } from "./strip-markdown";

package/src/utils/match-pattern.ts

@@ -31,8 +31,8 @@ export function matchPattern(
   const params: Record<string, string> = {};
 
   for (let i = 0; i < patternParts.length; i++) {
-    const pat = patternParts[i];
-    const val = pathParts[i];
+    const pat = patternParts[i]!;
+    const val = pathParts[i]!;
 
     if (pat.startsWith(":")) {
       params[pat.slice(1)] = val;

package/src/utils/strip-markdown.ts

@@ -0,0 +1,59 @@
+import { toString } from "mdast-util-to-string";
+import { toMarkdown } from "mdast-util-to-markdown";
+import type { Root, RootContent, List } from "mdast";
+
+export interface StripMarkdownOptions {
+  preserveLists?: boolean;
+}
+
+/**
+ * Strip markdown syntax from an MDAST, returning plain text.
+ * Headings, paragraphs, code blocks, blockquotes, etc. become plain text.
+ * With preserveLists: true, list items retain their bullet/number formatting.
+ */
+export function stripMarkdown(
+  ast: Root,
+  options: StripMarkdownOptions = {}
+): string {
+  const { preserveLists = false } = options;
+
+  return ast.children
+    .map((node: RootContent) => {
+      if (preserveLists && (node.type === "list")) {
+        return renderList(node as List, 0).trimEnd();
+      }
+      return toString(node);
+    })
+    .filter(Boolean)
+    .join("\n\n");
+}
+
+function renderList(list: List, depth: number): string {
+  const indent = "  ".repeat(depth);
+  let index = list.start ?? 1;
+
+  return list.children
+    .map((item) => {
+      const bullet = list.ordered ? `${index++}.` : "-";
+      const lines: string[] = [];
+
+      let firstContent = true;
+      for (const child of item.children) {
+        if (child.type === "list") {
+          lines.push(renderList(child as List, depth + 1));
+        } else {
+          const text = toString(child);
+          if (!text) continue;
+          if (firstContent) {
+            lines.push(`${indent}${bullet} ${text}`);
+            firstContent = false;
+          } else {
+            lines.push(`${indent}  ${text}`);
+          }
+        }
+      }
+
+      return lines.join("\n");
+    })
+    .join("\n");
+}