@mmnto/totem 0.1.0

package/dist/ingest/pipeline.js

@@ -0,0 +1,79 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { createChunker } from '../chunkers/chunker.js';
+import { createEmbedder } from '../embedders/embedder.js';
+import { LanceStore } from '../store/lance-store.js';
+import { getChangedFiles, resolveFiles } from './file-resolver.js';
+const EMBED_BATCH_SIZE = 100;
+export async function runSync(config, options) {
+    const { projectRoot, incremental, onProgress } = options;
+    const log = onProgress ?? (() => { });
+    // 1. Create embedder
+    log('Initializing embedding provider...');
+    const embedder = createEmbedder(config.embedding);
+    // 2. Connect to store
+    const storePath = path.join(projectRoot, config.lanceDir);
+    const store = new LanceStore(storePath, embedder);
+    await store.connect();
+    // 3. Resolve files to process
+    const allFiles = resolveFiles(config.targets, projectRoot, config.ignorePatterns);
+    let filesToProcess;
+    if (incremental) {
+        const changedPaths = options.changedFiles ?? getChangedFiles(projectRoot, 'HEAD~1', log);
+        if (changedPaths === null) {
+            log('Git diff failed, falling back to full sync...');
+            await store.reset();
+            filesToProcess = allFiles;
+            log(`Full sync (fallback): ${filesToProcess.length} files to process`);
+        }
+        else {
+            const changedSet = new Set(changedPaths);
+            filesToProcess = allFiles.filter((f) => changedSet.has(f.relativePath));
+            log(`Incremental sync: ${filesToProcess.length} changed files (of ${allFiles.length} total)`);
+        }
+    }
+    else {
+        log('Full sync: resetting index...');
+        await store.reset();
+        filesToProcess = allFiles;
+        log(`Full sync: ${filesToProcess.length} files to process`);
+    }
+    // 4. Chunk all files
+    const allChunks = [];
+    for (const file of filesToProcess) {
+        log(`Chunking ${file.relativePath}...`);
+        let content;
+        try {
+            content = fs.readFileSync(file.absolutePath, 'utf-8');
+        }
+        catch (err) {
+            log(`  Skipping (read error: ${err instanceof Error ? err.message : String(err)}): ${file.relativePath}`);
+            continue;
+        }
+        const chunker = createChunker(file.target.strategy);
+        const chunks = chunker.chunk(content, file.relativePath, file.target.type);
+        if (chunks.length === 0) {
+            log(`  No chunks extracted from ${file.relativePath}`);
+            continue;
+        }
+        // For incremental: delete old chunks for this file before inserting new ones
+        if (incremental) {
+            await store.deleteByFile(file.relativePath);
+        }
+        allChunks.push(...chunks);
+        log(`  ${chunks.length} chunks from ${file.relativePath}`);
+    }
+    // 5. Embed and store in batches
+    log(`Embedding ${allChunks.length} chunks...`);
+    for (let i = 0; i < allChunks.length; i += EMBED_BATCH_SIZE) {
+        const batch = allChunks.slice(i, i + EMBED_BATCH_SIZE);
+        await store.insert(batch);
+        log(`  Embedded ${Math.min(i + EMBED_BATCH_SIZE, allChunks.length)}/${allChunks.length}`);
+    }
+    log(`Sync complete: ${allChunks.length} chunks from ${filesToProcess.length} files`);
+    return {
+        chunksProcessed: allChunks.length,
+        filesProcessed: filesToProcess.length,
+    };
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/ingest/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEnE,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAmB,EACnB,OAAoB;IAEpB,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACzD,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAErC,qBAAqB;IACrB,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAElD,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IAEtB,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;IAClF,IAAI,cAA8B,CAAC;IAEnC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzF,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,GAAG,CAAC,+CAA+C,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,cAAc,GAAG,QAAQ,CAAC;YAC1B,GAAG,CAAC,yBAAyB,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YACzC,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,GAAG,CAAC,qBAAqB,cAAc,CAAC,MAAM,sBAAsB,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACrC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,cAAc,GAAG,QAAQ,CAAC;QAC1B,GAAG,CAAC,cAAc,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC9D,CAAC;IAED,qBAAqB;IACrB,MAAM,SAAS,GAAY,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,GAAG,CAAC,YAAY,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;QAExC,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CACD,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACrG,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,8BAA8B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QAED,6EAA6E;QAC7E,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC1B,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,gBAAgB,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,gCAAgC;IAChC,GAAG,CAAC,aAAa,SAAS,CAAC,MAAM,YAAY,CAAC,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,GAAG,CAAC,kBAAkB,SAAS,CAAC,MAAM,gBAAgB,cAAc,CAAC,MAAM,QAAQ,CAAC,CAAC;IAErF,OAAO;QACL,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,cAAc,EAAE,cAAc,CAAC,MAAM;KACtC,CAAC;AACJ,CAAC"}

package/dist/ingest/sync.d.ts

@@ -0,0 +1,4 @@
+export type { ResolvedFile } from './file-resolver.js';
+export { getChangedFiles, resolveFiles } from './file-resolver.js';
+export { runSync } from './pipeline.js';
+//# sourceMappingURL=sync.d.ts.map

package/dist/ingest/sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC"}

package/dist/ingest/sync.js

@@ -0,0 +1,3 @@
+export { getChangedFiles, resolveFiles } from './file-resolver.js';
+export { runSync } from './pipeline.js';
+//# sourceMappingURL=sync.js.map

package/dist/ingest/sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC"}

package/dist/store/lance-schema.d.ts

@@ -0,0 +1,42 @@
+import { z } from 'zod';
+/**
+ * Zod schema for validating a stored chunk before insertion.
+ * The vector field is not validated by Zod (handled by LanceDB).
+ */
+export declare const StoredChunkSchema: z.ZodObject<{
+    id: z.ZodString;
+    content: z.ZodString;
+    contextPrefix: z.ZodString;
+    filePath: z.ZodString;
+    type: z.ZodEnum<["code", "session_log", "spec"]>;
+    strategy: z.ZodEnum<["typescript-ast", "markdown-heading", "session-log", "schema-file", "test-file"]>;
+    label: z.ZodString;
+    startLine: z.ZodNumber;
+    endLine: z.ZodNumber;
+    metadata: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "code" | "session_log" | "spec";
+    strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
+    content: string;
+    contextPrefix: string;
+    filePath: string;
+    label: string;
+    startLine: number;
+    endLine: number;
+    id: string;
+    metadata: string;
+}, {
+    type: "code" | "session_log" | "spec";
+    strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
+    content: string;
+    contextPrefix: string;
+    filePath: string;
+    label: string;
+    startLine: number;
+    endLine: number;
+    id: string;
+    metadata: string;
+}>;
+/** The LanceDB table name used for all totem data. */
+export declare const TOTEM_TABLE_NAME = "totem_chunks";
+//# sourceMappingURL=lance-schema.d.ts.map

package/dist/store/lance-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lance-schema.d.ts","sourceRoot":"","sources":["../../src/store/lance-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;GAGG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAW5B,CAAC;AAEH,sDAAsD;AACtD,eAAO,MAAM,gBAAgB,iBAAiB,CAAC"}

package/dist/store/lance-schema.js

@@ -0,0 +1,21 @@
+import { z } from 'zod';
+import { ChunkStrategySchema, ContentTypeSchema } from '../config-schema.js';
+/**
+ * Zod schema for validating a stored chunk before insertion.
+ * The vector field is not validated by Zod (handled by LanceDB).
+ */
+export const StoredChunkSchema = z.object({
+    id: z.string(),
+    content: z.string(),
+    contextPrefix: z.string(),
+    filePath: z.string(),
+    type: ContentTypeSchema,
+    strategy: ChunkStrategySchema,
+    label: z.string(),
+    startLine: z.number().int(),
+    endLine: z.number().int(),
+    metadata: z.string(), // JSON-stringified
+});
+/** The LanceDB table name used for all totem data. */
+export const TOTEM_TABLE_NAME = 'totem_chunks';
+//# sourceMappingURL=lance-schema.js.map

package/dist/store/lance-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lance-schema.js","sourceRoot":"","sources":["../../src/store/lance-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE7E;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IACzB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,IAAI,EAAE,iBAAiB;IACvB,QAAQ,EAAE,mBAAmB;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACzB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,mBAAmB;CAC1C,CAAC,CAAC;AAEH,sDAAsD;AACtD,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC"}

package/dist/store/lance-store.d.ts

@@ -0,0 +1,27 @@
+import type { Embedder } from '../embedders/embedder.js';
+import type { Chunk, SearchOptions, SearchResult } from '../types.js';
+export declare class LanceStore {
+    private db;
+    private table;
+    private dbPath;
+    private embedder;
+    constructor(dbPath: string, embedder: Embedder);
+    /** Connect to LanceDB. Must be called before any other operations. */
+    connect(): Promise<void>;
+    /** Insert chunks into the store. Embeds them first. */
+    insert(chunks: Chunk[]): Promise<void>;
+    /** Semantic vector search with optional type filtering. */
+    search(options: SearchOptions): Promise<SearchResult[]>;
+    /** Delete all chunks from a specific file (for incremental re-index). */
+    deleteByFile(filePath: string): Promise<void>;
+    /** Drop the entire table. Used for full re-index. */
+    reset(): Promise<void>;
+    /** Re-open the LanceDB connection, picking up rebuilt files after a full sync. */
+    reconnect(): Promise<void>;
+    /** Return stats about the current index. */
+    stats(): Promise<{
+        totalChunks: number;
+        byType: Record<string, number>;
+    }>;
+}
+//# sourceMappingURL=lance-store.d.ts.map

package/dist/store/lance-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lance-store.d.ts","sourceRoot":"","sources":["../../src/store/lance-store.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAe,MAAM,aAAa,CAAC;AAGnF,qBAAa,UAAU;IACrB,OAAO,CAAC,EAAE,CAAmC;IAC7C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAW;gBAEf,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;IAK9C,sEAAsE;IAChE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAS9B,uDAAuD;IACjD,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B5C,2DAA2D;IACrD,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA0B7D,yEAAyE;IACnE,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD,qDAAqD;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B,kFAAkF;IAC5E,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAMhC,4CAA4C;IACtC,KAAK,IAAI,OAAO,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;CAchF"}

package/dist/store/lance-store.js

@@ -0,0 +1,106 @@
+import { randomUUID } from 'node:crypto';
+import * as lancedb from '@lancedb/lancedb';
+import { TOTEM_TABLE_NAME } from './lance-schema.js';
+export class LanceStore {
+    db = null;
+    table = null;
+    dbPath;
+    embedder;
+    constructor(dbPath, embedder) {
+        this.dbPath = dbPath;
+        this.embedder = embedder;
+    }
+    /** Connect to LanceDB. Must be called before any other operations. */
+    async connect() {
+        this.db = await lancedb.connect(this.dbPath);
+        const tableNames = await this.db.tableNames();
+        if (tableNames.includes(TOTEM_TABLE_NAME)) {
+            this.table = await this.db.openTable(TOTEM_TABLE_NAME);
+        }
+    }
+    /** Insert chunks into the store. Embeds them first. */
+    async insert(chunks) {
+        if (chunks.length === 0)
+            return;
+        // Build the text to embed: contextPrefix + content
+        const textsToEmbed = chunks.map((c) => `${c.contextPrefix} ${c.content}`);
+        const vectors = await this.embedder.embed(textsToEmbed);
+        const rows = chunks.map((chunk, i) => ({
+            id: randomUUID(),
+            content: chunk.content,
+            contextPrefix: chunk.contextPrefix,
+            filePath: chunk.filePath,
+            type: chunk.type,
+            strategy: chunk.strategy,
+            label: chunk.label,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            metadata: JSON.stringify(chunk.metadata),
+            vector: vectors[i],
+        }));
+        const data = rows;
+        if (!this.table) {
+            this.table = await this.db.createTable(TOTEM_TABLE_NAME, data);
+        }
+        else {
+            await this.table.add(data);
+        }
+    }
+    /** Semantic vector search with optional type filtering. */
+    async search(options) {
+        if (!this.table)
+            return [];
+        const maxResults = options.maxResults ?? 5;
+        const [queryVector] = await this.embedder.embed([options.query]);
+        let query = this.table.vectorSearch(queryVector).limit(maxResults);
+        if (options.typeFilter) {
+            query = query.where(`type = '${options.typeFilter.replace(/'/g, "''")}'`);
+        }
+        const results = await query.toArray();
+        return results.map((row) => ({
+            content: row['content'],
+            contextPrefix: row['contextPrefix'],
+            filePath: row['filePath'],
+            type: row['type'],
+            label: row['label'],
+            score: row['_distance'] != null ? 1 / (1 + row['_distance']) : 0,
+            metadata: JSON.parse(row['metadata'] || '{}'),
+        }));
+    }
+    /** Delete all chunks from a specific file (for incremental re-index). */
+    async deleteByFile(filePath) {
+        if (!this.table)
+            return;
+        await this.table.delete(`filePath = '${filePath.replace(/'/g, "''")}'`);
+    }
+    /** Drop the entire table. Used for full re-index. */
+    async reset() {
+        if (!this.db)
+            return;
+        const tableNames = await this.db.tableNames();
+        if (tableNames.includes(TOTEM_TABLE_NAME)) {
+            await this.db.dropTable(TOTEM_TABLE_NAME);
+        }
+        this.table = null;
+    }
+    /** Re-open the LanceDB connection, picking up rebuilt files after a full sync. */
+    async reconnect() {
+        this.db = null;
+        this.table = null;
+        await this.connect();
+    }
+    /** Return stats about the current index. */
+    async stats() {
+        if (!this.table)
+            return { totalChunks: 0, byType: {} };
+        const totalChunks = await this.table.countRows();
+        const typeRows = await this.table.query().select(['type']).toArray();
+        const byType = {};
+        for (const row of typeRows) {
+            const t = row['type'];
+            byType[t] = (byType[t] ?? 0) + 1;
+        }
+        return { totalChunks, byType };
+    }
+}
+//# sourceMappingURL=lance-store.js.map

package/dist/store/lance-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lance-store.js","sourceRoot":"","sources":["../../src/store/lance-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAK5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,OAAO,UAAU;IACb,EAAE,GAA8B,IAAI,CAAC;IACrC,KAAK,GAAyB,IAAI,CAAC;IACnC,MAAM,CAAS;IACf,QAAQ,CAAW;IAE3B,YAAY,MAAc,EAAE,QAAkB;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,MAAM,CAAC,MAAe;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,mDAAmD;QACnD,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAExD,MAAM,IAAI,GAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,EAAE,EAAE,UAAU,EAAE;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE;SACpB,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,IAA4C,CAAC;QAE1D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAE3B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAE3C,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAEjE,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAY,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEpE,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAA4B,EAAE,EAAE,CAAC,CAAC;YACpD,OAAO,EAAE,GAAG,CAAC,SAAS,CAAW;YACjC,aAAa,EAAE,GAAG,CAAC,eAAe,CAAW;YAC7C,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAW;YACnC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAgB;YAChC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAW;YAC7B,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAI,GAAG,CAAC,WAAW,CAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,GAAG,CAAC,UAAU,CAAY,IAAI,IAAI,CAA2B;SACpF,CAAC,CAAC,CAAC;IACN,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,YAAY,CAAC,QAAgB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;IAED,4CAA4C;IAC5C,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEvD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAEjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACrE,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAChC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;IACjC,CAAC;CACF"}

package/dist/types.d.ts

@@ -0,0 +1,75 @@
+import type { ChunkStrategy, ContentType } from './config-schema.js';
+/**
+ * A single chunk produced by any chunker.
+ * This is the unit that gets embedded and stored.
+ */
+export interface Chunk {
+    /** The text content to embed */
+    content: string;
+    /** Contextual prefix prepended before embedding */
+    contextPrefix: string;
+    /** Source file path (relative to project root) */
+    filePath: string;
+    /** Content type for metadata filtering */
+    type: ContentType;
+    /** Chunking strategy that produced this chunk */
+    strategy: ChunkStrategy;
+    /** Human-readable identifier (function name, heading path, etc.) */
+    label: string;
+    /** Start line in original file (1-indexed) */
+    startLine: number;
+    /** End line in original file (1-indexed) */
+    endLine: number;
+    /** Optional structured metadata (frontmatter fields, etc.) */
+    metadata: Record<string, string>;
+}
+/**
+ * A stored record in LanceDB — chunk + its embedding vector.
+ */
+export interface StoredChunk {
+    id: string;
+    content: string;
+    contextPrefix: string;
+    filePath: string;
+    type: string;
+    strategy: string;
+    label: string;
+    startLine: number;
+    endLine: number;
+    metadata: string;
+    vector: number[];
+}
+/**
+ * Result returned from a search query.
+ */
+export interface SearchResult {
+    content: string;
+    contextPrefix: string;
+    filePath: string;
+    type: ContentType;
+    label: string;
+    score: number;
+    metadata: Record<string, string>;
+}
+/**
+ * Options for the sync operation.
+ */
+export interface SyncOptions {
+    /** Project root directory (where totem.config.ts lives) */
+    projectRoot: string;
+    /** If true, only process files changed since last sync */
+    incremental: boolean;
+    /** If provided, only sync files matching these changed paths */
+    changedFiles?: string[];
+    /** Callback for progress reporting */
+    onProgress?: (message: string) => void;
+}
+/**
+ * Options for search queries.
+ */
+export interface SearchOptions {
+    query: string;
+    typeFilter?: ContentType;
+    maxResults?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAErE;;;GAGG;AACH,MAAM,WAAW,KAAK;IACpB,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAEhB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IAEtB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,0CAA0C;IAC1C,IAAI,EAAE,WAAW,CAAC;IAElB,iDAAiD;IACjD,QAAQ,EAAE,aAAa,CAAC;IAExB,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IAEd,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAElB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAEhB,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IAEpB,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;IAErB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,sCAAsC;IACtC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@mmnto/totem",
+  "version": "0.1.0",
+  "description": "Persistent memory and context layer for AI agents",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist",
+    "lint": "eslint src/",
+    "test": "echo \"No tests yet\" && exit 0"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.13.0",
+    "glob": "^11.0.0",
+    "openai": "^4.77.0",
+    "remark-frontmatter": "^5.0.0",
+    "remark-parse": "^11.0.0",
+    "typescript": "^5.7.0",
+    "unified": "^11.0.0",
+    "yaml": "^2.4.0",
+    "zod": "^3.24.0"
+  },
+  "devDependencies": {
+    "@types/mdast": "^4.0.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "author": "mmnto-ai",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mmnto-ai/totem.git",
+    "directory": "packages/core"
+  },
+  "homepage": "https://github.com/mmnto-ai/totem",
+  "bugs": {
+    "url": "https://github.com/mmnto-ai/totem/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}