maskweaver 0.9.3 → 0.9.5

package/dist/memory/chunking.js

@@ -1,13 +1,4 @@
-/**
- * Text Chunking - Break text into manageable pieces
- *
- * Simple line-based chunking with overlap.
- * Preserves context at chunk boundaries.
- */
 import { CONFIG, hashText, determineSource } from './core.js';
-/**
- * Chunk text into overlapping segments.
- */
 export function chunkText(text, filePath) {
     const { maxTokens, overlapTokens, charsPerToken } = CONFIG.chunking;
     const source = determineSource(filePath);
@@ -16,32 +7,28 @@ export function chunkText(text, filePath) {
     const maxChars = maxTokens * charsPerToken;
     const overlapChars = overlapTokens * charsPerToken;
     let currentChunkLines = [];
-    let currentChunkStart = 1; // 1-based line number
+    let currentChunkStart = 1;
     let currentLength = 0;
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        const lineLength = line.length + 1; // +1 for newline
+        const lineLength = line.length + 1;
         if (currentLength + lineLength <= maxChars) {
             currentChunkLines.push(line);
             currentLength += lineLength;
         }
         else {
-            // Save current chunk
             if (currentChunkLines.length > 0) {
                 const chunkText = currentChunkLines.join('\n');
                 if (chunkText.trim().length > 0) {
                     chunks.push(createChunk(filePath, currentChunkStart, currentChunkStart + currentChunkLines.length - 1, chunkText, source));
                 }
             }
-            // Calculate overlap
             const overlapLines = calculateOverlapLines(currentChunkLines, overlapChars);
-            // Start new chunk
             currentChunkStart = i + 1 - overlapLines.length;
             currentChunkLines = [...overlapLines, line];
             currentLength = currentChunkLines.join('\n').length;
         }
     }
-    // Save last chunk
     if (currentChunkLines.length > 0) {
         const chunkText = currentChunkLines.join('\n');
         if (chunkText.trim().length > 0) {
@@ -50,9 +37,6 @@ export function chunkText(text, filePath) {
     }
     return chunks;
 }
-/**
- * Calculate overlap lines from end of chunk.
- */
 function calculateOverlapLines(lines, overlapChars) {
     const result = [];
     let length = 0;
@@ -65,9 +49,6 @@ function calculateOverlapLines(lines, overlapChars) {
     }
     return result;
 }
-/**
- * Create chunk object.
- */
 function createChunk(path, startLine, endLine, text, source) {
     return {
         path,
@@ -78,27 +59,17 @@ function createChunk(path, startLine, endLine, text, source) {
         source,
     };
 }
-/**
- * Estimate tokens in text.
- */
 export function estimateTokens(text) {
     if (!text || text.length === 0)
         return 0;
-    // Korean characters: ~2 tokens per char
     const koreanChars = (text.match(/[\uAC00-\uD7A3]/g) || []).length;
-    // English words: ~1.3 tokens per word
     const englishWords = (text.match(/[a-zA-Z]+/g) || []).length;
-    // Other chars
     const otherChars = text.length - koreanChars - (englishWords * 5);
     const koreanTokens = koreanChars * 2;
     const englishTokens = englishWords * 1.3;
     const otherTokens = Math.max(0, otherChars) / 4;
     return Math.ceil(koreanTokens + englishTokens + otherTokens);
 }
-/**
- * Split text into sentences.
- * Handles both English and Korean.
- */
 export function splitIntoSentences(text) {
     const sentenceEnders = /([.!?。！？])\s+/g;
     const sentences = [];
@@ -114,9 +85,6 @@ export function splitIntoSentences(text) {
     return sentences.filter(s => s.length > 0);
 }
 const HEADER_REGEX = /^(#{1,6})\s+(.+)$/;
-/**
- * Parse markdown into sections.
- */
 export function parseMarkdownSections(content) {
     const lines = content.split('\n');
     const sections = [];
@@ -128,7 +96,6 @@ export function parseMarkdownSections(content) {
         const lineNumber = i + 1;
         const headerMatch = line.match(HEADER_REGEX);
         if (headerMatch) {
-            // Save previous section
             if (currentSection !== null || currentLines.length > 0) {
                 sections.push({
                     header: currentSection?.header || '',
@@ -138,7 +105,6 @@ export function parseMarkdownSections(content) {
                     endLine: lineNumber - 1
                 });
             }
-            // Start new section
             currentSection = {
                 header: headerMatch[2],
                 content: '',
@@ -153,7 +119,6 @@ export function parseMarkdownSections(content) {
             currentLines.push(line);
         }
     }
-    // Save last section
     if (currentLines.length > 0) {
         sections.push({
             header: currentSection?.header || '',

package/dist/memory/core.d.ts

@@ -1,8 +1,3 @@
-/**
- * Memory Core - Simple utilities for text processing
- *
- * Clear is better than clever.
- */
 export declare const CONFIG: {
     readonly chunking: {
         readonly maxTokens: 400;
@@ -36,34 +31,10 @@ export interface SearchResult {
     score: number;
     matchType: 'vector' | 'text' | 'hybrid';
 }
-/**
- * Hash text using djb2 algorithm.
- * Simple, fast, good enough for our use case.
- */
 export declare function hashText(text: string): string;
-/**
- * Estimate token count.
- * Rough approximation: 4 chars ≈ 1 token
- */
 export declare function estimateTokens(text: string): number;
-/**
- * Determine source type from file path.
- */
 export declare function determineSource(filePath: string): SourceType;
-/**
- * Calculate cosine similarity between two vectors.
- * Single pass for efficiency.
- *
- * Returns 0 on dimension mismatch instead of throwing,
- * to prevent a single bad embedding from crashing the entire search.
- */
 export declare function cosineSimilarity(a: number[], b: number[]): number;
-/**
- * Float32 version - more memory efficient.
- *
- * Returns 0 on dimension mismatch instead of throwing,
- * to prevent a single bad embedding from crashing the entire search.
- */
 export declare function cosineSimilarityFloat32(a: Float32Array, b: Float32Array): number;
 export declare function toFloat32Array(embedding: number[]): Float32Array;
 export declare function embeddingToBlob(embedding: number[]): Buffer;

package/dist/memory/core.js

@@ -1,12 +1,4 @@
-/**
- * Memory Core - Simple utilities for text processing
- *
- * Clear is better than clever.
- */
 import { join } from "path";
-// ============================================================================
-// Configuration
-// ============================================================================
 export const CONFIG = {
     chunking: {
         maxTokens: 400,
@@ -23,31 +15,17 @@ export const CONFIG = {
         defaultMinScore: 0.35,
     },
 };
-// ============================================================================
-// Text Processing
-// ============================================================================
-/**
- * Hash text using djb2 algorithm.
- * Simple, fast, good enough for our use case.
- */
 export function hashText(text) {
     let hash = 5381;
     for (let i = 0; i < text.length; i++) {
         hash = ((hash << 5) + hash) + text.charCodeAt(i);
-        hash = hash & hash; // Convert to 32bit integer
+        hash = hash & hash;
     }
     return Math.abs(hash).toString(16);
 }
-/**
- * Estimate token count.
- * Rough approximation: 4 chars ≈ 1 token
- */
 export function estimateTokens(text) {
     return Math.ceil(text.length / CONFIG.chunking.charsPerToken);
 }
-/**
- * Determine source type from file path.
- */
 export function determineSource(filePath) {
     const lower = filePath.toLowerCase().replace(/\\/g, '/');
     if (lower.includes('/daily/') || (lower.endsWith('.md') && /\d{4}-\d{2}-\d{2}/.test(lower))) {
@@ -67,18 +45,7 @@ export function determineSource(filePath) {
     }
     return 'code';
 }
-// ============================================================================
-// Similarity
-// ============================================================================
-// Deduplicated warning: only log dimension mismatch once per session
 let _dimensionMismatchWarned = false;
-/**
- * Calculate cosine similarity between two vectors.
- * Single pass for efficiency.
- *
- * Returns 0 on dimension mismatch instead of throwing,
- * to prevent a single bad embedding from crashing the entire search.
- */
 export function cosineSimilarity(a, b) {
     if (a.length === 0 || b.length === 0)
         return 0;
@@ -104,12 +71,6 @@ export function cosineSimilarity(a, b) {
     const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
     return magnitude === 0 ? 0 : dotProduct / magnitude;
 }
-/**
- * Float32 version - more memory efficient.
- *
- * Returns 0 on dimension mismatch instead of throwing,
- * to prevent a single bad embedding from crashing the entire search.
- */
 export function cosineSimilarityFloat32(a, b) {
     if (a.length === 0 || b.length === 0)
         return 0;
@@ -135,9 +96,6 @@ export function cosineSimilarityFloat32(a, b) {
     const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
     return magnitude === 0 ? 0 : dotProduct / magnitude;
 }
-// ============================================================================
-// Embedding Conversion
-// ============================================================================
 export function toFloat32Array(embedding) {
     return new Float32Array(embedding);
 }
@@ -152,9 +110,6 @@ export function blobToNumberArray(blob) {
     const float32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
     return Array.from(float32);
 }
-// ============================================================================
-// Date Utilities
-// ============================================================================
 export function getTodayFileName() {
     const now = new Date();
     const year = now.getFullYear();
@@ -168,9 +123,6 @@ export function dateToFileName(date) {
     const day = String(date.getDate()).padStart(2, '0');
     return `${year}-${month}-${day}.md`;
 }
-// ============================================================================
-// Path Utilities
-// ============================================================================
 export function getMemoryPath(type, basePath = process.cwd()) {
     const memoryDir = join(basePath, CONFIG.paths.memoryDir);
     switch (type) {
@@ -194,9 +146,6 @@ export function getDbPath(basePath = process.cwd()) {
 export function getDataDir(basePath = process.cwd()) {
     return join(basePath, CONFIG.paths.dataDir);
 }
-// ============================================================================
-// Utilities
-// ============================================================================
 export function sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }

package/dist/memory/index.d.ts

@@ -1,8 +1,3 @@
-/**
- * @maskweaver/memory
- *
- * Memory system with embeddings and vector search
- */
 export type { SourceType, MemoryType, Chunk, SearchResult, } from "./core.js";
 export { CONFIG, hashText, estimateTokens, determineSource, cosineSimilarity, cosineSimilarityFloat32, toFloat32Array, embeddingToBlob, blobToEmbedding, blobToNumberArray, getTodayFileName, dateToFileName, getMemoryPath, getDbPath, getDataDir, sleep, getConfig, } from "./core.js";
 export type { Embedding, ProviderType, HealthCheckResult, ProviderConfig, IEmbeddingProvider, } from "./providers/types.js";

package/dist/memory/index.js

@@ -1,21 +1,11 @@
-/**
- * @maskweaver/memory
- *
- * Memory system with embeddings and vector search
- */
 export { CONFIG, hashText, estimateTokens, determineSource, cosineSimilarity, cosineSimilarityFloat32, toFloat32Array, embeddingToBlob, blobToEmbedding, blobToNumberArray, getTodayFileName, dateToFileName, getMemoryPath, getDbPath, getDataDir, sleep, getConfig, } from "./core.js";
 export { OllamaProvider } from "./providers/ollama.js";
 export { OpenAIProvider } from "./providers/openai.js";
 export { VoyageProvider } from "./providers/voyage.js";
 export { OpenRouterProvider } from "./providers/openrouter.js";
 export { TextOnlyProvider } from "./providers/text-only.js";
-// Provider factory
 export { createProvider, selectBestProvider, getDefaultConfigs, } from "./providers/factory.js";
-// Chunking
 export { chunkText, parseMarkdownSections, splitIntoSentences } from "./chunking.js";
-// Storage
 export { MemoryDatabase, initDatabase, getDatabase, tryGetDatabase, isSqliteAvailable, getSqliteUnavailableReason, upsertChunk, searchByVector, searchByText, deleteChunksByPath, getChunksByPath, } from "./store/sqlite.js";
-// Search
 export { hybridSearch } from "./search/hybrid.js";
-// Indexing
 export { indexFile, reindexFile, indexAllMemoryFiles, classifySource, } from "./indexer.js";

package/dist/memory/indexer.d.ts

@@ -1,8 +1,3 @@
-/**
- * File Indexer - Process markdown files into searchable chunks
- *
- * Read file → Parse sections → Chunk → Hash → Embed → Store
- */
 import { type SourceType } from './core.js';
 type EmbeddingProvider = (text: string) => Promise<number[]>;
 export interface MarkdownSection {
@@ -12,25 +7,9 @@ export interface MarkdownSection {
     startLine: number;
     endLine: number;
 }
-/**
- * Parse markdown into sections.
- */
 export declare function parseMarkdownSections(content: string): MarkdownSection[];
-/**
- * Classify source from file path.
- */
 export declare function classifySource(filePath: string): SourceType;
-/**
- * Index a single file.
- * Only re-embeds chunks that changed (based on hash).
- */
 export declare function indexFile(filePath: string, getEmbedding: EmbeddingProvider, basePath?: string): Promise<void>;
-/**
- * Reindex file (delete all chunks and reindex).
- */
 export declare function reindexFile(filePath: string, getEmbedding: EmbeddingProvider, basePath?: string): Promise<void>;
-/**
- * Index all memory files in directory.
- */
 export declare function indexAllMemoryFiles(baseDir: string, getEmbedding: EmbeddingProvider, basePath?: string): Promise<void>;
 export {};

package/dist/memory/indexer.js

@@ -1,16 +1,8 @@
-/**
- * File Indexer - Process markdown files into searchable chunks
- *
- * Read file → Parse sections → Chunk → Hash → Embed → Store
- */
 import { readFile } from 'fs/promises';
 import { chunkText } from './chunking.js';
 import { initDatabase, upsertChunk, deleteChunksByPath, getChunksByPath } from './store/sqlite.js';
 import { getDbPath } from './core.js';
 const HEADER_REGEX = /^(#{1,6})\s+(.+)$/;
-/**
- * Parse markdown into sections.
- */
 export function parseMarkdownSections(content) {
     const lines = content.split('\n');
     const sections = [];
@@ -22,7 +14,6 @@ export function parseMarkdownSections(content) {
         const lineNumber = i + 1;
         const headerMatch = line.match(HEADER_REGEX);
         if (headerMatch) {
-            // Save previous section
             if (currentSection !== null || currentLines.length > 0) {
                 sections.push({
                     header: currentSection?.header || '',
@@ -32,7 +23,6 @@ export function parseMarkdownSections(content) {
                     endLine: lineNumber - 1
                 });
             }
-            // Start new section
             currentSection = {
                 header: headerMatch[2],
                 content: '',
@@ -47,7 +37,6 @@ export function parseMarkdownSections(content) {
             currentLines.push(line);
         }
     }
-    // Save last section
     if (currentLines.length > 0) {
         sections.push({
             header: currentSection?.header || '',
@@ -59,9 +48,6 @@ export function parseMarkdownSections(content) {
     }
     return sections;
 }
-/**
- * Classify source from file path.
- */
 export function classifySource(filePath) {
     const normalized = filePath.replace(/\\/g, '/').toLowerCase();
     const fileName = normalized.split('/').pop() || '';
@@ -77,34 +63,17 @@ export function classifySource(filePath) {
         return 'daily';
     return 'memory';
 }
-// ============================================================================
-// File Processing
-// ============================================================================
-/**
- * Process markdown file into chunks.
- */
 async function processFile(filePath) {
     const content = await readFile(filePath, 'utf-8');
     return chunkText(content, filePath);
 }
-// ============================================================================
-// Indexing
-// ============================================================================
-/**
- * Index a single file.
- * Only re-embeds chunks that changed (based on hash).
- */
 export async function indexFile(filePath, getEmbedding, basePath) {
     console.log(`[Indexer] Processing: ${filePath}`);
-    // Initialize database
     await initDatabase(getDbPath(basePath));
-    // Process file
     const chunks = await processFile(filePath);
     console.log(`[Indexer] Generated ${chunks.length} chunks`);
-    // Get existing chunks for hash comparison
     const existingChunks = getChunksByPath(filePath);
     const existingHashes = new Set(existingChunks.map(c => c.hash));
-    // Process only changed chunks
     let newCount = 0;
     let skippedCount = 0;
     for (const chunk of chunks) {
@@ -112,22 +81,16 @@ export async function indexFile(filePath, getEmbedding, basePath) {
             skippedCount++;
             continue;
         }
-        // New or changed chunk - compute embedding
         const embedding = await getEmbedding(chunk.text);
         upsertChunk(chunk, embedding);
         newCount++;
     }
     console.log(`[Indexer] Completed: ${newCount} new, ${skippedCount} unchanged`);
 }
-/**
- * Reindex file (delete all chunks and reindex).
- */
 export async function reindexFile(filePath, getEmbedding, basePath) {
     console.log(`[Indexer] Reindexing: ${filePath}`);
     await initDatabase(getDbPath(basePath));
-    // Delete existing chunks
     deleteChunksByPath(filePath);
-    // Process and index all chunks
     const chunks = await processFile(filePath);
     for (const chunk of chunks) {
         const embedding = await getEmbedding(chunk.text);
@@ -135,13 +98,9 @@ export async function reindexFile(filePath, getEmbedding, basePath) {
     }
     console.log(`[Indexer] Reindexed ${chunks.length} chunks`);
 }
-/**
- * Index all memory files in directory.
- */
 export async function indexAllMemoryFiles(baseDir, getEmbedding, basePath) {
     console.log(`[Indexer] Scanning directory: ${baseDir}`);
     await initDatabase(getDbPath(basePath || baseDir));
-    // Main memory files
     const mainFiles = ['MEMORY.md', 'MASKS.md', 'RETROSPECT.md', 'USER.md'];
     let totalFiles = 0;
     for (const fileName of mainFiles) {
@@ -151,11 +110,8 @@ export async function indexAllMemoryFiles(baseDir, getEmbedding, basePath) {
             totalFiles++;
         }
         catch (error) {
-            // File doesn't exist or not readable
             console.log(`[Indexer] Skipping ${fileName}: ${error}`);
         }
     }
-    // TODO: Scan daily directory
-    // This requires glob or fs.readdir
     console.log(`[Indexer] Completed indexing ${totalFiles} files`);
 }

package/dist/memory/providers/examples.d.ts

@@ -1,8 +1,3 @@
-/**
- * Embedding Provider Usage Examples
- *
- * Practical examples showing how to use the provider system.
- */
 declare function example1_autoSelection(): Promise<void>;
 declare function example2_manualSelection(): Promise<void>;
 declare function example3_codeEmbedding(): Promise<void>;