@plur-ai/core 0.4.0 → 0.4.2

package/dist/{chunk-WPD4MPTT.js → chunk-KMVQYBNP.js}

@@ -62,32 +62,42 @@ function computeIdf(engrams, queryTokens) {
   }
   return idf;
 }
-function ftsScore(engram, queryTokens, idfWeights) {
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
   const allTerms = ftsTokenize(engramSearchText(engram));
   if (queryTokens.length === 0) return 0;
-  let weightedHits = 0;
-  let totalWeight = 0;
+  const docLen = allTerms.length;
+  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
+  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
+  let score = 0;
   for (const qt of queryTokens) {
-    const weight = idfWeights?.get(qt) ?? 1;
-    totalWeight += weight;
-    if (allTerms.some((t) => t.includes(qt) || qt.includes(t))) {
-      weightedHits += weight;
+    let effectiveIdf;
+    if (!idfWeights) {
+      effectiveIdf = 1;
+    } else if (hasNonZeroIdf) {
+      effectiveIdf = idfWeights.get(qt) ?? 0;
+      if (effectiveIdf === 0) continue;
+    } else {
+      effectiveIdf = 1;
     }
-  }
-  if (totalWeight === 0) {
-    let matches = 0;
-    for (const qt of queryTokens) {
-      if (allTerms.some((t) => t.includes(qt) || qt.includes(t))) matches++;
+    let tf = 0;
+    for (const t of allTerms) {
+      if (t.includes(qt) || qt.includes(t)) tf++;
     }
-    return matches / queryTokens.length;
+    if (tf === 0) continue;
+    const numerator = tf * (BM25_K1 + 1);
+    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
+    score += effectiveIdf * (numerator / denominator);
   }
-  return weightedHits / totalWeight;
+  return score;
 }
 function searchEngrams(engrams, query, limit = 20) {
   const queryTokens = ftsTokenize(query);
   if (queryTokens.length === 0) return [];
   const idfWeights = computeIdf(engrams, queryTokens);
-  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
+  const avgDocLength = engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
+  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
 }
 
 // src/embeddings.ts
@@ -97,7 +107,7 @@ import { createHash } from "crypto";
 
 // src/sync.ts
 import { execFileSync } from "child_process";
-import { existsSync, writeFileSync, renameSync, mkdirSync } from "fs";
+import { existsSync, writeFileSync, renameSync, mkdirSync, unlinkSync, statSync } from "fs";
 import { join, dirname } from "path";
 var GITIGNORE = `# PLUR \u2014 derived/cache files (regenerated automatically)
 embeddings/
@@ -249,6 +259,44 @@ function sync(root, remote) {
     files_changed: filesChanged
   };
 }
+function withLock(filePath, fn, options) {
+  const lockPath = filePath + ".lock";
+  const maxRetries = options?.maxRetries ?? 5;
+  const baseDelay = options?.baseDelay ?? 100;
+  const staleThreshold = options?.staleThreshold ?? 1e4;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      writeFileSync(lockPath, `${process.pid}`, { flag: "wx" });
+      break;
+    } catch (err) {
+      if (err.code !== "EEXIST") throw err;
+      try {
+        const stat = statSync(lockPath);
+        if (Date.now() - stat.mtimeMs > staleThreshold) {
+          unlinkSync(lockPath);
+          continue;
+        }
+      } catch {
+        continue;
+      }
+      if (attempt === maxRetries) {
+        throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
+      }
+      const delay = baseDelay * Math.pow(2, attempt);
+      const end = Date.now() + delay;
+      while (Date.now() < end) {
+      }
+    }
+  }
+  try {
+    return fn();
+  } finally {
+    try {
+      unlinkSync(lockPath);
+    } catch {
+    }
+  }
+}
 function atomicWrite(filePath, content) {
   const dir = dirname(filePath);
   if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
@@ -337,6 +385,7 @@ async function embeddingSearch(engrams, query, limit, storagePath) {
 export {
   getSyncStatus,
   sync,
+  withLock,
   atomicWrite,
   ftsTokenize,
   engramSearchText,

package/dist/{embeddings-Q76LNQ5B.js → embeddings-2IODIQAF.js}

@@ -2,7 +2,7 @@ import {
   cosineSimilarity,
   embed,
   embeddingSearch
-} from "./chunk-WPD4MPTT.js";
+} from "./chunk-KMVQYBNP.js";
 import "./chunk-2ZDO52B4.js";
 export {
   cosineSimilarity,

package/dist/index.d.ts

@@ -616,6 +616,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access?: boolean | undefined;
     }>>>;
     allow_secrets: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
+    index: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
 }, "strip", z.ZodTypeAny, {
     auto_learn?: boolean | undefined;
     auto_capture?: boolean | undefined;
@@ -629,6 +630,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access: boolean;
     } | undefined;
     allow_secrets?: boolean | undefined;
+    index?: boolean | undefined;
 }, {
     auto_learn?: boolean | undefined;
     auto_capture?: boolean | undefined;
@@ -642,6 +644,7 @@ declare const PlurConfigSchema: z.ZodObject<{
         co_access?: boolean | undefined;
     } | undefined;
     allow_secrets?: boolean | undefined;
+    index?: boolean | undefined;
 }>;
 type PlurConfig = z.infer<typeof PlurConfigSchema>;
 
@@ -1164,9 +1167,36 @@ interface PlurPaths {
     packs: string;
     exchange: string;
     config: string;
+    db: string;
 }
 declare function detectPlurStorage(explicitPath?: string): PlurPaths;
 
+declare class IndexedStorage {
+    private dbPath;
+    private engramsPath;
+    private db;
+    constructor(engramsPath: string, dbPath: string);
+    private getDb;
+    /** Load all engrams from SQLite index. Auto-rebuilds if db missing. */
+    loadAll(): Engram[];
+    /** Load engrams with SQL-level filtering. */
+    loadFiltered(filter: {
+        status?: string;
+        scope?: string;
+        domain?: string;
+    }): Engram[];
+    /** Count engrams with optional status filter. */
+    count(filter?: {
+        status?: string;
+    }): number;
+    /** Sync SQLite index from YAML source of truth. */
+    syncFromYaml(): void;
+    /** Drop and rebuild the entire index from YAML. */
+    reindex(): void;
+    /** Close the database connection. */
+    close(): void;
+}
+
 /**
  * Non-blocking version check against npm registry.
  * Caches result in memory — one fetch per process lifetime.
@@ -1212,6 +1242,7 @@ interface StatusResult {
 declare class Plur {
     private paths;
     private config;
+    private indexedStorage;
     constructor(options?: {
         path?: string;
     });
@@ -1263,6 +1294,15 @@ declare class Plur {
     updateEngram(updated: Engram): boolean;
     /** Set engram status to 'retired'. */
     forget(id: string, reason?: string): void;
+    /** Remove retired engrams from storage. Returns count of removed and remaining. */
+    compact(): {
+        removed: number;
+        remaining: number;
+    };
+    /** Rebuild SQLite index from YAML source of truth. Only works when index: true. */
+    reindex(): void;
+    /** Sync SQLite index after YAML write (no-op if index disabled) */
+    private _syncIndex;
     /** Capture an episodic memory. */
     capture(summary: string, context?: CaptureContext): Episode;
     /** Query the episode timeline. */
@@ -1294,4 +1334,4 @@ declare class Plur {
     status(): StatusResult;
 }
 
-export { type AlignmentResult, type Association, type CaptureContext, type DomainCoverage, DomainCoverageSchema, type Engram, type EngramCluster, type Episode, type EvidenceEntry, EvidenceEntrySchema, type ExtractOptions, type ExtractionResult, type Falsification, FalsificationSchema, type HierarchyPosition, HierarchyPositionSchema, type IngestCandidate, type IngestOptions, type InjectOptions, type InjectionResult, type KnowledgeAnchor, type LearnContext, type LlmFunction, type MemberAlignment, type MetaConfidence, MetaConfidenceSchema, type MetaField, MetaFieldSchema, PLATITUDE_PATTERNS, type PackManifest, Plur, type PlurConfig, type PlurPaths, type RecallOptions, type RelationalAnalysis, type RelationalTriple, SessionBreadcrumbs, type StatusResult, type StructuralTemplate, StructuralTemplateSchema, type SyncResult, type SyncStatus, type TimelineQuery, type TypedRole, type ValidationResult, type VersionCheckResult, alignCluster, analyzeStructure, checkForUpdate, classifyPolarity, clearVersionCache, clusterByStructure, computeConfidence, computeMetaConfidence, confidenceBand, detectPlurStorage, detectSecrets, engramSearchText, extractMetaEngrams, formulateMetaEngram, generateGuardrails, getCachedUpdateCheck, isPlatitude, organizeHierarchy, tokenSimilarity, validateMetaEngram };
+export { type AlignmentResult, type Association, type CaptureContext, type DomainCoverage, DomainCoverageSchema, type Engram, type EngramCluster, type Episode, type EvidenceEntry, EvidenceEntrySchema, type ExtractOptions, type ExtractionResult, type Falsification, FalsificationSchema, type HierarchyPosition, HierarchyPositionSchema, IndexedStorage, type IngestCandidate, type IngestOptions, type InjectOptions, type InjectionResult, type KnowledgeAnchor, type LearnContext, type LlmFunction, type MemberAlignment, type MetaConfidence, MetaConfidenceSchema, type MetaField, MetaFieldSchema, PLATITUDE_PATTERNS, type PackManifest, Plur, type PlurConfig, type PlurPaths, type RecallOptions, type RelationalAnalysis, type RelationalTriple, SessionBreadcrumbs, type StatusResult, type StructuralTemplate, StructuralTemplateSchema, type SyncResult, type SyncStatus, type TimelineQuery, type TypedRole, type ValidationResult, type VersionCheckResult, alignCluster, analyzeStructure, checkForUpdate, classifyPolarity, clearVersionCache, clusterByStructure, computeConfidence, computeMetaConfidence, confidenceBand, detectPlurStorage, detectSecrets, engramSearchText, extractMetaEngrams, formulateMetaEngram, generateGuardrails, getCachedUpdateCheck, isPlatitude, organizeHierarchy, tokenSimilarity, validateMetaEngram };

package/dist/index.js

@@ -6,8 +6,9 @@ import {
   ftsTokenize,
   getSyncStatus,
   searchEngrams,
-  sync
-} from "./chunk-WPD4MPTT.js";
+  sync,
+  withLock
+} from "./chunk-KMVQYBNP.js";
 import "./chunk-2ZDO52B4.js";
 
 // src/storage.ts
@@ -26,98 +27,71 @@ function detectPlurStorage(explicitPath) {
     candidates: join(root, "candidates.yaml"),
     packs: packsDir,
     exchange: join(root, "exchange"),
-    config: join(root, "config.yaml")
+    config: join(root, "config.yaml"),
+    db: join(root, "engrams.db")
   };
 }
 
-// src/config.ts
-import { existsSync as existsSync2, readFileSync } from "fs";
-import yaml from "js-yaml";
-
-// src/schemas/config.ts
-import { z } from "zod";
-var PlurConfigSchema = z.object({
-  auto_learn: z.boolean().default(true),
-  auto_capture: z.boolean().default(true),
-  injection_budget: z.number().default(2e3),
-  decay_enabled: z.boolean().default(true),
-  decay_threshold: z.number().default(0.15),
-  packs: z.array(z.string()).default([]),
-  injection: z.object({
-    spread_cap: z.number().default(3),
-    spread_budget: z.number().default(480),
-    co_access: z.boolean().default(true)
-  }).default({}),
-  allow_secrets: z.boolean().default(false)
-}).partial();
-
-// src/config.ts
-function loadConfig(configPath) {
-  if (!existsSync2(configPath)) return PlurConfigSchema.parse({});
-  try {
-    const raw = yaml.load(readFileSync(configPath, "utf8"));
-    return PlurConfigSchema.parse(raw ?? {});
-  } catch {
-    return PlurConfigSchema.parse({});
-  }
-}
+// src/storage-indexed.ts
+import { existsSync as existsSync3 } from "fs";
+import { createRequire } from "module";
 
 // src/engrams.ts
 import * as fs from "fs";
-import * as yaml2 from "js-yaml";
+import * as yaml from "js-yaml";
 
 // src/schemas/engram.ts
-import { z as z2 } from "zod";
-var ActivationSchema = z2.object({
-  retrieval_strength: z2.number().min(0).max(1),
-  storage_strength: z2.number().min(0).max(1),
-  frequency: z2.number().int().min(0),
-  last_accessed: z2.string()
+import { z } from "zod";
+var ActivationSchema = z.object({
+  retrieval_strength: z.number().min(0).max(1),
+  storage_strength: z.number().min(0).max(1),
+  frequency: z.number().int().min(0),
+  last_accessed: z.string()
 });
-var KnowledgeTypeSchema = z2.object({
-  memory_class: z2.enum(["semantic", "episodic", "procedural", "metacognitive"]),
-  cognitive_level: z2.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"])
+var KnowledgeTypeSchema = z.object({
+  memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]),
+  cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"])
 });
-var KnowledgeAnchorSchema = z2.object({
-  path: z2.string(),
-  relevance: z2.enum(["primary", "supporting", "example"]).default("supporting"),
-  snippet: z2.string().max(200).optional(),
-  snippet_extracted_at: z2.string().optional()
+var KnowledgeAnchorSchema = z.object({
+  path: z.string(),
+  relevance: z.enum(["primary", "supporting", "example"]).default("supporting"),
+  snippet: z.string().max(200).optional(),
+  snippet_extracted_at: z.string().optional()
 });
-var AssociationSchema = z2.object({
-  target_type: z2.enum(["engram", "document"]),
-  target: z2.string(),
-  strength: z2.number().min(0).max(0.95),
-  type: z2.enum(["semantic", "temporal", "causal", "co_accessed"]),
-  updated_at: z2.string().optional()
+var AssociationSchema = z.object({
+  target_type: z.enum(["engram", "document"]),
+  target: z.string(),
+  strength: z.number().min(0).max(0.95),
+  type: z.enum(["semantic", "temporal", "causal", "co_accessed"]),
+  updated_at: z.string().optional()
 });
-var DualCodingSchema = z2.object({
-  example: z2.string().optional(),
-  analogy: z2.string().optional()
+var DualCodingSchema = z.object({
+  example: z.string().optional(),
+  analogy: z.string().optional()
 }).refine(
   (d) => d.example || d.analogy,
   "At least one of example or analogy must be provided"
 );
-var RelationsSchema = z2.object({
-  broader: z2.array(z2.string()).default([]),
-  narrower: z2.array(z2.string()).default([]),
-  related: z2.array(z2.string()).default([]),
-  conflicts: z2.array(z2.string()).default([])
+var RelationsSchema = z.object({
+  broader: z.array(z.string()).default([]),
+  narrower: z.array(z.string()).default([]),
+  related: z.array(z.string()).default([]),
+  conflicts: z.array(z.string()).default([])
 });
-var ProvenanceSchema = z2.object({
-  origin: z2.string(),
-  chain: z2.array(z2.string()).default([]),
-  signature: z2.string().nullable().default(null),
-  license: z2.string().default("cc-by-sa-4.0")
+var ProvenanceSchema = z.object({
+  origin: z.string(),
+  chain: z.array(z.string()).default([]),
+  signature: z.string().nullable().default(null),
+  license: z.string().default("cc-by-sa-4.0")
 });
-var FeedbackSignalsSchema = z2.object({
-  positive: z2.number().int().default(0),
-  negative: z2.number().int().default(0),
-  neutral: z2.number().int().default(0)
+var FeedbackSignalsSchema = z.object({
+  positive: z.number().int().default(0),
+  negative: z.number().int().default(0),
+  neutral: z.number().int().default(0)
 });
-var EntityRefSchema = z2.object({
-  name: z2.string(),
-  type: z2.enum([
+var EntityRefSchema = z.object({
+  name: z.string(),
+  type: z.enum([
     "person",
     "organization",
     "technology",
@@ -129,56 +103,56 @@ var EntityRefSchema = z2.object({
     "standard",
     "other"
   ]),
-  uri: z2.string().url().optional()
+  uri: z.string().url().optional()
 });
-var TemporalSchema = z2.object({
-  learned_at: z2.string(),
-  valid_from: z2.string().optional(),
-  valid_until: z2.string().optional(),
-  ingested_at: z2.string().optional()
+var TemporalSchema = z.object({
+  learned_at: z.string(),
+  valid_from: z.string().optional(),
+  valid_until: z.string().optional(),
+  ingested_at: z.string().optional()
 });
-var UsageStatsSchema = z2.object({
-  injections: z2.number().int().default(0),
-  hits: z2.number().int().default(0),
-  misses: z2.number().int().default(0),
-  last_hit_at: z2.string().optional()
+var UsageStatsSchema = z.object({
+  injections: z.number().int().default(0),
+  hits: z.number().int().default(0),
+  misses: z.number().int().default(0),
+  last_hit_at: z.string().optional()
 });
-var EpisodicFieldsSchema = z2.object({
-  emotional_weight: z2.number().int().min(1).max(10).default(5),
-  confidence: z2.number().int().min(1).max(10).default(5),
-  trigger_context: z2.string().optional(),
-  journal_ref: z2.string().optional()
+var EpisodicFieldsSchema = z.object({
+  emotional_weight: z.number().int().min(1).max(10).default(5),
+  confidence: z.number().int().min(1).max(10).default(5),
+  trigger_context: z.string().optional(),
+  journal_ref: z.string().optional()
 });
-var ExchangeMetadataSchema = z2.object({
-  fitness_score: z2.number().min(0).max(1).optional(),
-  environmental_diversity: z2.number().int().default(0),
-  adoption_count: z2.number().int().default(0),
-  contradiction_rate: z2.number().min(0).max(1).default(0)
+var ExchangeMetadataSchema = z.object({
+  fitness_score: z.number().min(0).max(1).optional(),
+  environmental_diversity: z.number().int().default(0),
+  adoption_count: z.number().int().default(0),
+  contradiction_rate: z.number().min(0).max(1).default(0)
 });
-var EngramSchema = z2.object({
+var EngramSchema = z.object({
   // Identity
-  id: z2.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/),
-  version: z2.number().int().min(1).default(2),
-  status: z2.enum(["active", "dormant", "retired", "candidate"]),
-  consolidated: z2.boolean().default(false),
-  type: z2.enum(["behavioral", "terminological", "procedural", "architectural"]),
-  scope: z2.string(),
-  visibility: z2.enum(["private", "public", "template"]).default("private"),
+  id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/),
+  version: z.number().int().min(1).default(2),
+  status: z.enum(["active", "dormant", "retired", "candidate"]),
+  consolidated: z.boolean().default(false),
+  type: z.enum(["behavioral", "terminological", "procedural", "architectural"]),
+  scope: z.string(),
+  visibility: z.enum(["private", "public", "template"]).default("private"),
   // Content
-  statement: z2.string().min(1),
-  rationale: z2.string().optional(),
-  contraindications: z2.array(z2.string()).optional(),
+  statement: z.string().min(1),
+  rationale: z.string().optional(),
+  contraindications: z.array(z.string()).optional(),
   // Lineage
-  source: z2.string().optional(),
-  source_patterns: z2.array(z2.string()).optional(),
-  derivation_count: z2.number().int().min(0).default(1),
-  pack: z2.string().nullable().default(null),
-  abstract: z2.string().nullable().default(null),
-  derived_from: z2.string().nullable().default(null),
+  source: z.string().optional(),
+  source_patterns: z.array(z.string()).optional(),
+  derivation_count: z.number().int().min(0).default(1),
+  pack: z.string().nullable().default(null),
+  abstract: z.string().nullable().default(null),
+  derived_from: z.string().nullable().default(null),
   // Classification
   knowledge_type: KnowledgeTypeSchema.optional(),
-  domain: z2.string().optional(),
-  tags: z2.array(z2.string()).default([]),
+  domain: z.string().optional(),
+  tags: z.array(z.string()).default([]),
   // Activation (ACT-R model)
   activation: ActivationSchema.default({
     retrieval_strength: 0.7,
@@ -188,8 +162,8 @@ var EngramSchema = z2.object({
   }),
   // Relations & grounding
   relations: RelationsSchema.optional(),
-  associations: z2.array(AssociationSchema).default([]),
-  knowledge_anchors: z2.array(KnowledgeAnchorSchema).default([]),
+  associations: z.array(AssociationSchema).default([]),
+  knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]),
   dual_coding: DualCodingSchema.optional(),
   // Provenance
   provenance: ProvenanceSchema.optional(),
@@ -197,7 +171,7 @@ var EngramSchema = z2.object({
   feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }),
   // === NEW OPTIONAL FIELDS (v2.1) ===
   /** Typed entity references extracted from statement. Enables graph queries. */
-  entities: z2.array(EntityRefSchema).optional(),
+  entities: z.array(EntityRefSchema).optional(),
   /** Temporal validity window. When is this knowledge true? */
   temporal: TemporalSchema.optional(),
   /** Automatic usage tracking. Injections, hits, misses. */
@@ -207,33 +181,33 @@ var EngramSchema = z2.object({
   /** Exchange marketplace metadata: fitness, adoption, diversity. */
   exchange: ExchangeMetadataSchema.optional(),
   /** Extensible key-value data for domain-specific fields. */
-  structured_data: z2.record(z2.string(), z2.unknown()).optional(),
+  structured_data: z.record(z.string(), z.unknown()).optional(),
   /** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */
-  polarity: z2.enum(["do", "dont"]).nullable().default(null)
+  polarity: z.enum(["do", "dont"]).nullable().default(null)
 });
 
 // src/schemas/pack.ts
-import { z as z3 } from "zod";
-var PackManifestSchema = z3.object({
-  name: z3.string(),
-  version: z3.string(),
-  description: z3.string().optional(),
-  creator: z3.string().optional(),
-  license: z3.string().default("cc-by-sa-4.0"),
-  tags: z3.array(z3.string()).default([]),
-  metadata: z3.object({
-    id: z3.string().optional(),
-    injection_policy: z3.enum(["on_match", "on_request", "always"]).default("on_match"),
-    match_terms: z3.array(z3.string()).default([]),
-    domain: z3.string().optional(),
-    engram_count: z3.number().optional()
+import { z as z2 } from "zod";
+var PackManifestSchema = z2.object({
+  name: z2.string(),
+  version: z2.string(),
+  description: z2.string().optional(),
+  creator: z2.string().optional(),
+  license: z2.string().default("cc-by-sa-4.0"),
+  tags: z2.array(z2.string()).default([]),
+  metadata: z2.object({
+    id: z2.string().optional(),
+    injection_policy: z2.enum(["on_match", "on_request", "always"]).default("on_match"),
+    match_terms: z2.array(z2.string()).default([]),
+    domain: z2.string().optional(),
+    engram_count: z2.number().optional()
   }).optional(),
-  "x-datacore": z3.object({
-    id: z3.string(),
-    injection_policy: z3.enum(["on_match", "on_request"]),
-    match_terms: z3.array(z3.string()).default([]),
-    domain: z3.string().optional(),
-    engram_count: z3.number().int().min(0)
+  "x-datacore": z2.object({
+    id: z2.string(),
+    injection_policy: z2.enum(["on_match", "on_request"]),
+    match_terms: z2.array(z2.string()).default([]),
+    domain: z2.string().optional(),
+    engram_count: z2.number().int().min(0)
   }).optional()
 });
 
@@ -260,7 +234,7 @@ var logger = {
 function loadEngrams(filePath) {
   if (!fs.existsSync(filePath)) return [];
   try {
-    const raw = yaml2.load(fs.readFileSync(filePath, "utf8"));
+    const raw = yaml.load(fs.readFileSync(filePath, "utf8"));
     if (!raw?.engrams || !Array.isArray(raw.engrams)) return [];
     const valid = [];
     let skipped = 0;
@@ -277,14 +251,14 @@ function loadEngrams(filePath) {
   }
 }
 function saveEngrams(filePath, engrams) {
-  const content = yaml2.dump({ engrams }, { lineWidth: 120, noRefs: true, quotingType: '"' });
+  const content = yaml.dump({ engrams }, { lineWidth: 120, noRefs: true, quotingType: '"' });
   atomicWrite(filePath, content);
 }
 function parseSkillMdFrontmatter(filePath) {
   const content = fs.readFileSync(filePath, "utf8");
   const match = content.match(/^---\n([\s\S]*?)\n---/);
   if (!match) throw new Error(`No frontmatter found in ${filePath}`);
-  return yaml2.load(match[1]);
+  return yaml.load(match[1]);
 }
 function loadPack(packDir) {
   const skillMdPath = `${packDir}/SKILL.md`;
@@ -294,7 +268,7 @@ function loadPack(packDir) {
   if (fs.existsSync(skillMdPath)) {
     rawManifest = parseSkillMdFrontmatter(skillMdPath);
   } else if (fs.existsSync(manifestYamlPath)) {
-    rawManifest = yaml2.load(fs.readFileSync(manifestYamlPath, "utf8"));
+    rawManifest = yaml.load(fs.readFileSync(manifestYamlPath, "utf8"));
   } else {
     throw new Error(`No SKILL.md or manifest.yaml found in ${packDir}`);
   }
@@ -326,6 +300,166 @@ function generateEngramId(existing) {
   return `${prefix}${String(next).padStart(3, "0")}`;
 }
 
+// src/storage-indexed.ts
+var require2 = createRequire(import.meta.url);
+var Database = null;
+function getDatabase() {
+  if (!Database) {
+    try {
+      Database = require2("better-sqlite3");
+    } catch {
+      throw new Error(
+        "better-sqlite3 is required for index: true. Install it with: npm install better-sqlite3"
+      );
+    }
+  }
+  return Database;
+}
+var IndexedStorage = class {
+  dbPath;
+  engramsPath;
+  db = null;
+  constructor(engramsPath, dbPath) {
+    this.engramsPath = engramsPath;
+    this.dbPath = dbPath;
+  }
+  getDb() {
+    if (!this.db) {
+      const DB = getDatabase();
+      this.db = new DB(this.dbPath);
+      this.db.pragma("journal_mode = WAL");
+      this.db.exec(`
+        CREATE TABLE IF NOT EXISTS engrams (
+          id TEXT PRIMARY KEY,
+          status TEXT NOT NULL,
+          scope TEXT NOT NULL,
+          domain TEXT,
+          last_accessed TEXT,
+          data TEXT NOT NULL
+        );
+        CREATE INDEX IF NOT EXISTS idx_status ON engrams(status);
+        CREATE INDEX IF NOT EXISTS idx_scope ON engrams(scope);
+        CREATE INDEX IF NOT EXISTS idx_domain ON engrams(domain);
+      `);
+    }
+    return this.db;
+  }
+  /** Load all engrams from SQLite index. Auto-rebuilds if db missing. */
+  loadAll() {
+    if (!existsSync3(this.dbPath)) {
+      this.reindex();
+    }
+    const db = this.getDb();
+    const rows = db.prepare("SELECT data FROM engrams").all();
+    return rows.map((r) => JSON.parse(r.data));
+  }
+  /** Load engrams with SQL-level filtering. */
+  loadFiltered(filter) {
+    if (!existsSync3(this.dbPath)) {
+      this.reindex();
+    }
+    const db = this.getDb();
+    const conditions = [];
+    const params = [];
+    if (filter.status) {
+      conditions.push("status = ?");
+      params.push(filter.status);
+    }
+    if (filter.scope) {
+      conditions.push("(scope = 'global' OR scope = ? OR scope LIKE ? || '%')");
+      params.push(filter.scope, filter.scope);
+    }
+    if (filter.domain) {
+      conditions.push("domain LIKE ? || '%'");
+      params.push(filter.domain);
+    }
+    const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+    const rows = db.prepare(`SELECT data FROM engrams ${where}`).all(...params);
+    return rows.map((r) => JSON.parse(r.data));
+  }
+  /** Count engrams with optional status filter. */
+  count(filter) {
+    if (!existsSync3(this.dbPath)) {
+      this.reindex();
+    }
+    const db = this.getDb();
+    if (filter?.status) {
+      return db.prepare("SELECT COUNT(*) as c FROM engrams WHERE status = ?").get(filter.status).c;
+    }
+    return db.prepare("SELECT COUNT(*) as c FROM engrams").get().c;
+  }
+  /** Sync SQLite index from YAML source of truth. */
+  syncFromYaml() {
+    const engrams = loadEngrams(this.engramsPath);
+    const db = this.getDb();
+    const upsert = db.prepare(`
+      INSERT OR REPLACE INTO engrams (id, status, scope, domain, last_accessed, data)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+    const deleteStmt = db.prepare("DELETE FROM engrams WHERE id = ?");
+    const yamlIds = new Set(engrams.map((e) => e.id));
+    const dbIds = new Set(
+      db.prepare("SELECT id FROM engrams").all().map((r) => r.id)
+    );
+    const tx = db.transaction(() => {
+      for (const e of engrams) {
+        upsert.run(e.id, e.status, e.scope, e.domain ?? null, e.activation.last_accessed, JSON.stringify(e));
+      }
+      for (const id of dbIds) {
+        if (!yamlIds.has(id)) deleteStmt.run(id);
+      }
+    });
+    tx();
+  }
+  /** Drop and rebuild the entire index from YAML. */
+  reindex() {
+    this.close();
+    const db = this.getDb();
+    db.exec("DELETE FROM engrams");
+    this.syncFromYaml();
+  }
+  /** Close the database connection. */
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+};
+
+// src/config.ts
+import { existsSync as existsSync4, readFileSync as readFileSync2 } from "fs";
+import yaml2 from "js-yaml";
+
+// src/schemas/config.ts
+import { z as z3 } from "zod";
+var PlurConfigSchema = z3.object({
+  auto_learn: z3.boolean().default(true),
+  auto_capture: z3.boolean().default(true),
+  injection_budget: z3.number().default(2e3),
+  decay_enabled: z3.boolean().default(true),
+  decay_threshold: z3.number().default(0.15),
+  packs: z3.array(z3.string()).default([]),
+  injection: z3.object({
+    spread_cap: z3.number().default(3),
+    spread_budget: z3.number().default(480),
+    co_access: z3.boolean().default(true)
+  }).default({}),
+  allow_secrets: z3.boolean().default(false),
+  index: z3.boolean().default(false)
+}).partial();
+
+// src/config.ts
+function loadConfig(configPath) {
+  if (!existsSync4(configPath)) return PlurConfigSchema.parse({});
+  try {
+    const raw = yaml2.load(readFileSync2(configPath, "utf8"));
+    return PlurConfigSchema.parse(raw ?? {});
+  } catch {
+    return PlurConfigSchema.parse({});
+  }
+}
+
 // src/decay.ts
 var DECAY_RATE = 0.05;
 var FLOOR = 0.05;
@@ -653,7 +787,7 @@ function selectAndSpread(ctx, personalEngrams, packs, config, embeddingBoosts) {
 }
 
 // src/episodes.ts
-import { existsSync as existsSync4, readFileSync as readFileSync3 } from "fs";
+import { existsSync as existsSync5, readFileSync as readFileSync3 } from "fs";
 import yaml3 from "js-yaml";
 function generateEpisodeId() {
   const ts = Date.now();
@@ -688,7 +822,7 @@ function queryTimeline(path2, query) {
   return episodes;
 }
 function loadEpisodes(path2) {
-  if (!existsSync4(path2)) return [];
+  if (!existsSync5(path2)) return [];
   try {
     const raw = yaml3.load(readFileSync3(path2, "utf8"));
     return Array.isArray(raw) ? raw : [];
@@ -1097,7 +1231,7 @@ async function computeSimilarityMatrix(templates) {
   const n = templates.length;
   const matrix = Array.from({ length: n }, () => Array(n).fill(0));
   try {
-    const { embed, cosineSimilarity } = await import("./embeddings-Q76LNQ5B.js");
+    const { embed, cosineSimilarity } = await import("./embeddings-2IODIQAF.js");
     const embeddings = [];
     for (const t of templates) {
       embeddings.push(await embed(t));
@@ -1832,9 +1966,13 @@ var INGEST_PATTERNS = [
 var Plur = class {
   paths;
   config;
+  indexedStorage = null;
   constructor(options) {
     this.paths = detectPlurStorage(options?.path);
     this.config = loadConfig(this.paths.config);
+    if (this.config.index) {
+      this.indexedStorage = new IndexedStorage(this.paths.engrams, this.paths.db);
+    }
   }
   /** Create engram, detect conflicts, save. Returns the created engram. */
   learn(statement, context) {
@@ -1844,48 +1982,51 @@ var Plur = class {
         throw new Error(`Secret detected in statement: ${secrets[0].pattern}. Use config.allow_secrets to override.`);
       }
     }
-    const engrams = loadEngrams(this.paths.engrams);
-    const id = generateEngramId(engrams);
-    const scope = context?.scope ?? "global";
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const conflictingEngrams = detectConflicts({ statement, scope }, engrams);
-    const conflictIds = conflictingEngrams.map((e) => e.id);
-    const engram = {
-      id,
-      version: 2,
-      status: "active",
-      consolidated: false,
-      type: context?.type ?? "behavioral",
-      scope,
-      visibility: "private",
-      statement,
-      source: context?.source,
-      domain: context?.domain,
-      activation: {
-        retrieval_strength: 0.7,
-        storage_strength: 1,
-        frequency: 0,
-        last_accessed: now.slice(0, 10)
-      },
-      feedback_signals: { positive: 0, negative: 0, neutral: 0 },
-      knowledge_anchors: [],
-      associations: [],
-      derivation_count: 1,
-      tags: [],
-      pack: null,
-      abstract: null,
-      derived_from: null,
-      polarity: null,
-      relations: conflictIds.length > 0 ? {
-        broader: [],
-        narrower: [],
-        related: [],
-        conflicts: conflictIds
-      } : void 0
-    };
-    engrams.push(engram);
-    saveEngrams(this.paths.engrams, engrams);
-    return engram;
+    return withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const id = generateEngramId(engrams);
+      const scope = context?.scope ?? "global";
+      const now = (/* @__PURE__ */ new Date()).toISOString();
+      const conflictingEngrams = detectConflicts({ statement, scope }, engrams);
+      const conflictIds = conflictingEngrams.map((e) => e.id);
+      const engram = {
+        id,
+        version: 2,
+        status: "active",
+        consolidated: false,
+        type: context?.type ?? "behavioral",
+        scope,
+        visibility: "private",
+        statement,
+        source: context?.source,
+        domain: context?.domain,
+        activation: {
+          retrieval_strength: 0.7,
+          storage_strength: 1,
+          frequency: 0,
+          last_accessed: now.slice(0, 10)
+        },
+        feedback_signals: { positive: 0, negative: 0, neutral: 0 },
+        knowledge_anchors: [],
+        associations: [],
+        derivation_count: 1,
+        tags: [],
+        pack: null,
+        abstract: null,
+        derived_from: null,
+        polarity: null,
+        relations: conflictIds.length > 0 ? {
+          broader: [],
+          narrower: [],
+          related: [],
+          conflicts: conflictIds
+        } : void 0
+      };
+      engrams.push(engram);
+      saveEngrams(this.paths.engrams, engrams);
+      this._syncIndex();
+      return engram;
+    });
   }
   /**
    * Search engrams, filter by scope/domain/strength, reactivate accessed.
@@ -1939,75 +2080,89 @@ var Plur = class {
   }
   /** Filter engrams by scope/domain/strength (shared by both modes) */
   _filterEngrams(options) {
-    let engrams = loadEngrams(this.paths.engrams);
-    engrams = engrams.filter((e) => e.status === "active");
+    let engrams;
+    if (this.indexedStorage) {
+      engrams = this.indexedStorage.loadFiltered({
+        status: "active",
+        scope: options?.scope,
+        domain: options?.domain
+      });
+    } else {
+      engrams = loadEngrams(this.paths.engrams);
+      engrams = engrams.filter((e) => e.status === "active");
+      if (options?.domain) {
+        engrams = engrams.filter((e) => e.domain?.startsWith(options.domain));
+      }
+      if (options?.scope) {
+        const scope = options.scope;
+        engrams = engrams.filter(
+          (e) => e.scope === "global" || e.scope === scope || e.scope.startsWith(scope)
+        );
+      }
+    }
     const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
     engrams = engrams.filter((e) => {
       if (e.temporal?.valid_until && e.temporal.valid_until < today) return false;
       if (e.temporal?.valid_from && e.temporal.valid_from > today) return false;
       return true;
     });
-    if (options?.domain) {
-      engrams = engrams.filter((e) => e.domain?.startsWith(options.domain));
-    }
     if (options?.min_strength !== void 0) {
       engrams = engrams.filter((e) => e.activation.retrieval_strength >= options.min_strength);
     }
-    if (options?.scope) {
-      const scope = options.scope;
-      engrams = engrams.filter(
-        (e) => e.scope === "global" || e.scope === scope || e.scope.startsWith(scope)
-      );
-    }
     return engrams;
   }
   /** Reactivate accessed engrams and update co-access associations */
   _reactivateResults(results) {
     if (results.length === 0) return;
-    const allEngrams = loadEngrams(this.paths.engrams);
-    const resultIds = new Set(results.map((e) => e.id));
-    const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
-    let modified = false;
-    for (const e of allEngrams) {
-      if (resultIds.has(e.id)) {
-        e.activation.retrieval_strength = reactivate(e.activation.retrieval_strength);
-        e.activation.last_accessed = today;
-        e.activation.frequency += 1;
-        modified = true;
+    withLock(this.paths.engrams, () => {
+      const allEngrams = loadEngrams(this.paths.engrams);
+      const resultIds = new Set(results.map((e) => e.id));
+      const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+      let modified = false;
+      for (const e of allEngrams) {
+        if (resultIds.has(e.id)) {
+          e.activation.retrieval_strength = reactivate(e.activation.retrieval_strength);
+          e.activation.last_accessed = today;
+          e.activation.frequency += 1;
+          modified = true;
+        }
       }
-    }
-    if (results.length >= 2 && this.config.injection?.co_access !== false) {
-      const topHalf = results.slice(0, Math.max(2, Math.ceil(results.length / 2)));
-      const topIds = topHalf.map((e) => e.id);
-      for (const sourceId of topIds) {
-        const source = allEngrams.find((e) => e.id === sourceId);
-        if (!source) continue;
-        for (const targetId of topIds) {
-          if (targetId === sourceId) continue;
-          const existing = source.associations.find(
-            (a) => a.type === "co_accessed" && a.target === targetId
-          );
-          if (existing) {
-            existing.strength = Math.min(0.95, existing.strength + 0.05);
-            existing.updated_at = today;
-            modified = true;
-          } else {
-            const coAccessCount = source.associations.filter((a) => a.type === "co_accessed").length;
-            if (coAccessCount < 5) {
-              source.associations.push({
-                target_type: "engram",
-                target: targetId,
-                type: "co_accessed",
-                strength: 0.3,
-                updated_at: today
-              });
+      if (results.length >= 2 && this.config.injection?.co_access !== false) {
+        const topHalf = results.slice(0, Math.max(2, Math.ceil(results.length / 2)));
+        const topIds = topHalf.map((e) => e.id);
+        for (const sourceId of topIds) {
+          const source = allEngrams.find((e) => e.id === sourceId);
+          if (!source) continue;
+          for (const targetId of topIds) {
+            if (targetId === sourceId) continue;
+            const existing = source.associations.find(
+              (a) => a.type === "co_accessed" && a.target === targetId
+            );
+            if (existing) {
+              existing.strength = Math.min(0.95, existing.strength + 0.05);
+              existing.updated_at = today;
               modified = true;
+            } else {
+              const coAccessCount = source.associations.filter((a) => a.type === "co_accessed").length;
+              if (coAccessCount < 5) {
+                source.associations.push({
+                  target_type: "engram",
+                  target: targetId,
+                  type: "co_accessed",
+                  strength: 0.3,
+                  updated_at: today
+                });
+                modified = true;
+              }
             }
           }
         }
       }
-    }
-    if (modified) saveEngrams(this.paths.engrams, allEngrams);
+      if (modified) {
+        saveEngrams(this.paths.engrams, allEngrams);
+        this._syncIndex();
+      }
+    });
   }
   /** Scored injection within token budget (BM25 only). Returns formatted strings. */
   inject(task, options) {
@@ -2066,56 +2221,96 @@ var Plur = class {
   }
   /** Update feedback_signals and adjust retrieval_strength. */
   feedback(id, signal) {
-    const engrams = loadEngrams(this.paths.engrams);
-    const engram = engrams.find((e) => e.id === id);
-    if (!engram) throw new Error(`Engram not found: ${id}`);
-    if (!engram.feedback_signals) {
-      engram.feedback_signals = { positive: 0, negative: 0, neutral: 0 };
-    }
-    engram.feedback_signals[signal] += 1;
-    if (signal === "positive") {
-      engram.activation.retrieval_strength = Math.min(1, engram.activation.retrieval_strength + 0.05);
-    } else if (signal === "negative") {
-      engram.activation.retrieval_strength = Math.max(0, engram.activation.retrieval_strength - 0.1);
-    }
-    saveEngrams(this.paths.engrams, engrams);
+    withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const engram = engrams.find((e) => e.id === id);
+      if (!engram) throw new Error(`Engram not found: ${id}`);
+      if (!engram.feedback_signals) {
+        engram.feedback_signals = { positive: 0, negative: 0, neutral: 0 };
+      }
+      engram.feedback_signals[signal] += 1;
+      if (signal === "positive") {
+        engram.activation.retrieval_strength = Math.min(1, engram.activation.retrieval_strength + 0.05);
+      } else if (signal === "negative") {
+        engram.activation.retrieval_strength = Math.max(0, engram.activation.retrieval_strength - 0.1);
+      }
+      saveEngrams(this.paths.engrams, engrams);
+      this._syncIndex();
+    });
   }
   /** Save extracted meta-engrams to the engram store. Skips IDs that already exist. */
   saveMetaEngrams(metas) {
-    const engrams = loadEngrams(this.paths.engrams);
-    const existingIds = new Set(engrams.map((e) => e.id));
-    let saved = 0;
-    let skipped = 0;
-    for (const meta of metas) {
-      if (existingIds.has(meta.id)) {
-        skipped++;
-      } else {
-        engrams.push(meta);
-        saved++;
+    return withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const existingIds = new Set(engrams.map((e) => e.id));
+      let saved = 0;
+      let skipped = 0;
+      for (const meta of metas) {
+        if (existingIds.has(meta.id)) {
+          skipped++;
+        } else {
+          engrams.push(meta);
+          saved++;
+        }
       }
-    }
-    if (saved > 0) saveEngrams(this.paths.engrams, engrams);
-    return { saved, skipped };
+      if (saved > 0) {
+        saveEngrams(this.paths.engrams, engrams);
+        this._syncIndex();
+      }
+      return { saved, skipped };
+    });
   }
   /** Update an existing engram in the store by ID. Returns true if found and updated. */
   updateEngram(updated) {
-    const engrams = loadEngrams(this.paths.engrams);
-    const idx = engrams.findIndex((e) => e.id === updated.id);
-    if (idx === -1) return false;
-    engrams[idx] = updated;
-    saveEngrams(this.paths.engrams, engrams);
-    return true;
+    return withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const idx = engrams.findIndex((e) => e.id === updated.id);
+      if (idx === -1) return false;
+      engrams[idx] = updated;
+      saveEngrams(this.paths.engrams, engrams);
+      this._syncIndex();
+      return true;
+    });
   }
   /** Set engram status to 'retired'. */
   forget(id, reason) {
-    const engrams = loadEngrams(this.paths.engrams);
-    const engram = engrams.find((e) => e.id === id);
-    if (!engram) throw new Error(`Engram not found: ${id}`);
-    engram.status = "retired";
-    if (reason && !engram.rationale) {
-      engram.rationale = `Retired: ${reason}`;
+    withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const engram = engrams.find((e) => e.id === id);
+      if (!engram) throw new Error(`Engram not found: ${id}`);
+      engram.status = "retired";
+      if (reason && !engram.rationale) {
+        engram.rationale = `Retired: ${reason}`;
+      }
+      saveEngrams(this.paths.engrams, engrams);
+      this._syncIndex();
+    });
+  }
+  /** Remove retired engrams from storage. Returns count of removed and remaining. */
+  compact() {
+    return withLock(this.paths.engrams, () => {
+      const engrams = loadEngrams(this.paths.engrams);
+      const active = engrams.filter((e) => e.status !== "retired");
+      const removed = engrams.length - active.length;
+      if (removed > 0) {
+        saveEngrams(this.paths.engrams, active);
+        this._syncIndex();
+      }
+      return { removed, remaining: active.length };
+    });
+  }
+  /** Rebuild SQLite index from YAML source of truth. Only works when index: true. */
+  reindex() {
+    if (!this.indexedStorage) {
+      this.indexedStorage = new IndexedStorage(this.paths.engrams, this.paths.db);
+    }
+    this.indexedStorage.reindex();
+  }
+  /** Sync SQLite index after YAML write (no-op if index disabled) */
+  _syncIndex() {
+    if (this.indexedStorage) {
+      this.indexedStorage.syncFromYaml();
     }
-    saveEngrams(this.paths.engrams, engrams);
   }
   /** Capture an episodic memory. */
   capture(summary, context) {
@@ -2196,6 +2391,7 @@ export {
   EvidenceEntrySchema,
   FalsificationSchema,
   HierarchyPositionSchema,
+  IndexedStorage,
   MetaConfidenceSchema,
   MetaFieldSchema,
   PLATITUDE_PATTERNS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plur-ai/core",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,9 +12,11 @@
     "zod": "^3.23.0"
   },
   "optionalDependencies": {
-    "@huggingface/transformers": "^3.8.1"
+    "@huggingface/transformers": "^3.8.1",
+    "better-sqlite3": "^11.0.0"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
     "@types/js-yaml": "^4.0.0",
     "@types/node": "^25.5.0"
   },