@plur-ai/core 0.4.0 → 0.4.1

package/dist/{chunk-WPD4MPTT.js → chunk-KMVQYBNP.js}

@@ -62,32 +62,42 @@ function computeIdf(engrams, queryTokens) {
   }
   return idf;
 }
-function ftsScore(engram, queryTokens, idfWeights) {
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
   const allTerms = ftsTokenize(engramSearchText(engram));
   if (queryTokens.length === 0) return 0;
-  let weightedHits = 0;
-  let totalWeight = 0;
+  const docLen = allTerms.length;
+  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
+  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
+  let score = 0;
   for (const qt of queryTokens) {
-    const weight = idfWeights?.get(qt) ?? 1;
-    totalWeight += weight;
-    if (allTerms.some((t) => t.includes(qt) || qt.includes(t))) {
-      weightedHits += weight;
+    let effectiveIdf;
+    if (!idfWeights) {
+      effectiveIdf = 1;
+    } else if (hasNonZeroIdf) {
+      effectiveIdf = idfWeights.get(qt) ?? 0;
+      if (effectiveIdf === 0) continue;
+    } else {
+      effectiveIdf = 1;
     }
-  }
-  if (totalWeight === 0) {
-    let matches = 0;
-    for (const qt of queryTokens) {
-      if (allTerms.some((t) => t.includes(qt) || qt.includes(t))) matches++;
+    let tf = 0;
+    for (const t of allTerms) {
+      if (t.includes(qt) || qt.includes(t)) tf++;
     }
-    return matches / queryTokens.length;
+    if (tf === 0) continue;
+    const numerator = tf * (BM25_K1 + 1);
+    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
+    score += effectiveIdf * (numerator / denominator);
   }
-  return weightedHits / totalWeight;
+  return score;
 }
 function searchEngrams(engrams, query, limit = 20) {
   const queryTokens = ftsTokenize(query);
   if (queryTokens.length === 0) return [];
   const idfWeights = computeIdf(engrams, queryTokens);
-  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
+  const avgDocLength = engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
+  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
 }
 
 // src/embeddings.ts
@@ -97,7 +107,7 @@ import { createHash } from "crypto";
 
 // src/sync.ts
 import { execFileSync } from "child_process";
-import { existsSync, writeFileSync, renameSync, mkdirSync } from "fs";
+import { existsSync, writeFileSync, renameSync, mkdirSync, unlinkSync, statSync } from "fs";
 import { join, dirname } from "path";
 var GITIGNORE = `# PLUR \u2014 derived/cache files (regenerated automatically)
 embeddings/
@@ -249,6 +259,44 @@ function sync(root, remote) {
     files_changed: filesChanged
   };
 }
+function withLock(filePath, fn, options) {
+  const lockPath = filePath + ".lock";
+  const maxRetries = options?.maxRetries ?? 5;
+  const baseDelay = options?.baseDelay ?? 100;
+  const staleThreshold = options?.staleThreshold ?? 1e4;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      writeFileSync(lockPath, `${process.pid}`, { flag: "wx" });
+      break;
+    } catch (err) {
+      if (err.code !== "EEXIST") throw err;
+      try {
+        const stat = statSync(lockPath);
+        if (Date.now() - stat.mtimeMs > staleThreshold) {
+          unlinkSync(lockPath);
+          continue;
+        }
+      } catch {
+        continue;
+      }
+      if (attempt === maxRetries) {
+        throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
+      }
+      const delay = baseDelay * Math.pow(2, attempt);
+      const end = Date.now() + delay;
+      while (Date.now() < end) {
+      }
+    }
+  }
+  try {
+    return fn();
+  } finally {
+    try {
+      unlinkSync(lockPath);
+    } catch {
+    }
+  }
+}
 function atomicWrite(filePath, content) {
   const dir = dirname(filePath);
   if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
@@ -337,6 +385,7 @@ async function embeddingSearch(engrams, query, limit, storagePath) {
 export {
   getSyncStatus,
   sync,
+  withLock,
   atomicWrite,
   ftsTokenize,
   engramSearchText,

package/dist/{embeddings-Q76LNQ5B.js → embeddings-2IODIQAF.js}

@@ -2,7 +2,7 @@ import {
   cosineSimilarity,
   embed,
   embeddingSearch
-} from "./chunk-WPD4MPTT.js";
+} from "./chunk-KMVQYBNP.js";
 import "./chunk-2ZDO52B4.js";
 export {
   cosineSimilarity,

package/dist/index.d.ts

@@ -616,6 +616,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access?: boolean | undefined;
     }>>>;
     allow_secrets: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
+    index: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
 }, "strip", z.ZodTypeAny, {
     auto_learn?: boolean | undefined;
     auto_capture?: boolean | undefined;
@@ -629,6 +630,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access: boolean;
     } | undefined;
     allow_secrets?: boolean | undefined;
+    index?: boolean | undefined;
 }, {
     auto_learn?: boolean | undefined;
     auto_capture?: boolean | undefined;
@@ -642,6 +644,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access?: boolean | undefined;
     } | undefined;
     allow_secrets?: boolean | undefined;
+    index?: boolean | undefined;
 }>;
 type PlurConfig = z.infer<typeof PlurConfigSchema>;
 
@@ -1164,9 +1167,36 @@ interface PlurPaths {
     packs: string;
     exchange: string;
     config: string;
+    db: string;
 }
 declare function detectPlurStorage(explicitPath?: string): PlurPaths;
 
+declare class IndexedStorage {
+    private dbPath;
+    private engramsPath;
+    private db;
+    constructor(engramsPath: string, dbPath: string);
+    private getDb;
+    /** Load all engrams from SQLite index. Auto-rebuilds if db missing. */
+    loadAll(): Engram[];
+    /** Load engrams with SQL-level filtering. */
+    loadFiltered(filter: {
+        status?: string;
+        scope?: string;
+        domain?: string;
+    }): Engram[];
+    /** Count engrams with optional status filter. */
+    count(filter?: {
+        status?: string;
+    }): number;
+    /** Sync SQLite index from YAML source of truth. */
+    syncFromYaml(): void;
+    /** Drop and rebuild the entire index from YAML. */
+    reindex(): void;
+    /** Close the database connection. */
+    close(): void;
+}
+
 /**
  * Non-blocking version check against npm registry.
  * Caches result in memory — one fetch per process lifetime.
@@ -1212,6 +1242,7 @@ interface StatusResult {
 declare class Plur {
     private paths;
     private config;
+    private indexedStorage;
     constructor(options?: {
         path?: string;
     });
@@ -1263,6 +1294,15 @@ declare class Plur {
     updateEngram(updated: Engram): boolean;
     /** Set engram status to 'retired'. */
     forget(id: string, reason?: string): void;
+    /** Remove retired engrams from storage. Returns count of removed and remaining. */
+    compact(): {
+        removed: number;
+        remaining: number;
+    };
+    /** Rebuild SQLite index from YAML source of truth. Only works when index: true. */
+    reindex(): void;
+    /** Sync SQLite index after YAML write (no-op if index disabled) */
+    private _syncIndex;
     /** Capture an episodic memory. */
     capture(summary: string, context?: CaptureContext): Episode;
     /** Query the episode timeline. */
@@ -1294,4 +1334,4 @@ declare class Plur {
     status(): StatusResult;
 }
 
-export { type AlignmentResult, type Association, type CaptureContext, type DomainCoverage, DomainCoverageSchema, type Engram, type EngramCluster, type Episode, type EvidenceEntry, EvidenceEntrySchema, type ExtractOptions, type ExtractionResult, type Falsification, FalsificationSchema, type HierarchyPosition, HierarchyPositionSchema, type IngestCandidate, type IngestOptions, type InjectOptions, type InjectionResult, type KnowledgeAnchor, type LearnContext, type LlmFunction, type MemberAlignment, type MetaConfidence, MetaConfidenceSchema, type MetaField, MetaFieldSchema, PLATITUDE_PATTERNS, type PackManifest, Plur, type PlurConfig, type PlurPaths, type RecallOptions, type RelationalAnalysis, type RelationalTriple, SessionBreadcrumbs, type StatusResult, type StructuralTemplate, StructuralTemplateSchema, type SyncResult, type SyncStatus, type TimelineQuery, type TypedRole, type ValidationResult, type VersionCheckResult, alignCluster, analyzeStructure, checkForUpdate, classifyPolarity, clearVersionCache, clusterByStructure, computeConfidence, computeMetaConfidence, confidenceBand, detectPlurStorage, detectSecrets, engramSearchText, extractMetaEngrams, formulateMetaEngram, generateGuardrails, getCachedUpdateCheck, isPlatitude, organizeHierarchy, tokenSimilarity, validateMetaEngram };
+export { type AlignmentResult, type Association, type CaptureContext, type DomainCoverage, DomainCoverageSchema, type Engram, type EngramCluster, type Episode, type EvidenceEntry, EvidenceEntrySchema, type ExtractOptions, type ExtractionResult, type Falsification, FalsificationSchema, type HierarchyPosition, HierarchyPositionSchema, IndexedStorage, type IngestCandidate, type IngestOptions, type InjectOptions, type InjectionResult, type KnowledgeAnchor, type LearnContext, type LlmFunction, type MemberAlignment, type MetaConfidence, MetaConfidenceSchema, type MetaField, MetaFieldSchema, PLATITUDE_PATTERNS, type PackManifest, Plur, type PlurConfig, type PlurPaths, type RecallOptions, type RelationalAnalysis, type RelationalTriple, SessionBreadcrumbs, type StatusResult, type StructuralTemplate, StructuralTemplateSchema, type SyncResult, type SyncStatus, type TimelineQuery, type TypedRole, type ValidationResult, type VersionCheckResult, alignCluster, analyzeStructure, checkForUpdate, classifyPolarity, clearVersionCache, clusterByStructure, computeConfidence, computeMetaConfidence, confidenceBand, detectPlurStorage, detectSecrets, engramSearchText, extractMetaEngrams, formulateMetaEngram, generateGuardrails, getCachedUpdateCheck, isPlatitude, organizeHierarchy, tokenSimilarity, validateMetaEngram };