@seanhogg/builderforce-memory 2026.7.14 → 2026.9.27

package/src/evermind/recall.ts

@@ -0,0 +1,122 @@
+/**
+ * evermind/recall.ts — the lexical recall rules for a project Evermind's taught memories.
+ *
+ * The gateway ranks a project's retained contributions against a query ("which learned
+ * memory would answer this?"), and the chat client decides which recalled memories an
+ * answer RECONCILES (restates enough of to supersede). Both must key on the SAME
+ * meaningful terms, so the tokenizer, stopword set and both rules live here, once.
+ *
+ * Pure and dependency-free: safe in a Worker, in Node, and in a browser bundle.
+ */
+
+/** Tiny code+English stopword set — dropped so recall keys on meaningful terms. */
+export const RECALL_STOPWORDS: ReadonlySet<string> = new Set([
+    'the', 'a', 'an', 'and', 'or', 'to', 'of', 'in', 'on', 'for', 'with', 'is', 'are',
+    'be', 'as', 'at', 'by', 'it', 'this', 'that', 'from', 'you', 'your', 'i', 'we',
+    'they', 'he', 'she', 'can', 'will', 'how', 'do', 'does', 'what', 'why', 'when',
+    'which', 'use', 'using', 'used', 'please', 'need', 'want', 'me', 'my', 'so', 'if',
+]);
+
+/** Lowercase, split on non-word runs, drop stopwords and 1-char tokens. */
+export function recallTokens(text: string): string[] {
+    return (text.toLowerCase().match(/[a-z0-9_]+/g) ?? []).filter((w) => w.length >= 2 && !RECALL_STOPWORDS.has(w));
+}
+
+/** One contribution the ranker scores (a subset of a retained-contribution entry). */
+export interface RecallScorable {
+    id: number;
+    kind: 'text' | 'delta';
+    version: number;
+    at: number;
+    weight: number;
+    prompt?: string;
+    text?: string;
+    /** The chat that contributed this memory, when one did. Absent means project-wide. */
+    chatId?: number;
+}
+
+/** A scored recall match — the entry plus its 0..1 lexical relevance to the query. */
+export interface RankedEvermindRecall extends RecallScorable {
+    /** Lexical relevance to the query, 0..1 (rounded to 3 dp). */
+    score: number;
+}
+
+function termFreq(tokens: string[]): Map<string, number> {
+    const m = new Map<string, number>();
+    for (const tok of tokens) m.set(tok, (m.get(tok) ?? 0) + 1);
+    return m;
+}
+
+/** Cosine similarity between two TF maps (0..1). Iterates the smaller map. */
+function tfCosine(a: Map<string, number>, b: Map<string, number>): number {
+    let na = 0;
+    for (const v of a.values()) na += v * v;
+    let nb = 0;
+    for (const v of b.values()) nb += v * v;
+    if (na === 0 || nb === 0) return 0;
+    const [small, big] = a.size <= b.size ? [a, b] : [b, a];
+    let dot = 0;
+    for (const [k, v] of small) {
+        const w = big.get(k);
+        if (w) dot += v * w;
+    }
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+/** Below this cosine a match is noise, not recall. */
+const MIN_RECALL_SCORE = 0.02;
+
+/**
+ * Rank contributions by lexical relevance to `query`, best first. Only entries carrying
+ * text can match (a weight delta has none); scores at or below the noise floor are
+ * dropped, and the result is capped at `limit` (default 8).
+ */
+export function rankEvermindRecall(
+    query: string,
+    recent: readonly RecallScorable[],
+    opts?: { limit?: number },
+): RankedEvermindRecall[] {
+    const q = termFreq(recallTokens(query));
+    if (q.size === 0) return [];
+    const scored: RankedEvermindRecall[] = [];
+    for (const e of recent) {
+        const hay = `${e.prompt ?? ''} ${e.text ?? ''}`.trim();
+        if (!hay) continue;
+        const score = tfCosine(q, termFreq(recallTokens(hay)));
+        if (score > MIN_RECALL_SCORE) scored.push({ ...e, score: Math.round(score * 1000) / 1000 });
+    }
+    scored.sort((a, b) => b.score - a.score || b.at - a.at);
+    return scored.slice(0, Math.max(1, opts?.limit ?? 8));
+}
+
+/** Stable, bounded (djb2, base36) hash of a recall query — keeps a cache key finite. */
+export function hashRecallQuery(query: string): string {
+    let h = 5381;
+    for (let i = 0; i < query.length; i++) h = ((h << 5) + h + query.charCodeAt(i)) | 0;
+    return (h >>> 0).toString(36);
+}
+
+/**
+ * Fraction of a recalled memory's meaningful tokens an answer must restate for the
+ * turn to count as RECONCILING (superseding) that memory.
+ */
+export const RECONCILE_OVERLAP = 0.6;
+
+/**
+ * How many recalled memories `answer` reconciles — restates enough of that the
+ * contributed turn supersedes them. A heuristic over token overlap; it surfaces the
+ * reconcile step and never gates learning.
+ */
+export function countReconciledMemories(items: ReadonlyArray<{ text: string }>, answer: string): number {
+    const ans = new Set(recallTokens(answer));
+    if (ans.size === 0) return 0;
+    let n = 0;
+    for (const it of items) {
+        const mem = new Set(recallTokens(it.text));
+        if (mem.size === 0) continue;
+        let hit = 0;
+        for (const tok of mem) if (ans.has(tok)) hit++;
+        if (hit / mem.size >= RECONCILE_OVERLAP) n++;
+    }
+    return n;
+}

package/src/index.ts

@@ -100,12 +100,15 @@ export type { SSMRuntimeOptions, GenerateOptions } from './runtime/SSMRuntime.js
 // ── Bridges ───────────────────────────────────────────────────────────────────
 export type { TransformerBridge, BridgeGenerateOptions } from './bridges/TransformerBridge.js';
 export { OpenAIBridge }    from './bridges/OpenAIBridge.js';
+export { VertexAIBridge, VertexAIEmbedder } from './bridges/VertexAIBridge.js';
 export { AnthropicBridge } from './bridges/AnthropicBridge.js';
 export { FetchBridge }     from './bridges/FetchBridge.js';
 export { CachingBridge }   from './bridges/CachingBridge.js';
 export { SemanticCachingBridge } from './bridges/SemanticCachingBridge.js';
 export { ResponseCache, buildCacheKey } from './bridges/ResponseCache.js';
+export type { BridgeCallInfo }         from './bridges/TransformerBridge.js';
 export type { OpenAIBridgeOptions }    from './bridges/OpenAIBridge.js';
+export type { VertexAIBridgeOptions, VertexAIEmbedderOptions } from './bridges/VertexAIBridge.js';
 export type { AnthropicBridgeOptions } from './bridges/AnthropicBridge.js';
 export type { FetchBridgeOptions }     from './bridges/FetchBridge.js';
 export type { CachingBridgeOptions }   from './bridges/CachingBridge.js';
@@ -130,6 +133,11 @@ export { cosineSimilarity, jaccardSimilarity, tokenize } from './similarity/inde
 export {
     chunkText,
     bm25Search,
+    bm25Idf,
+    bm25LengthNorm,
+    bm25TermScore,
+    BM25_DEFAULT_K1,
+    BM25_DEFAULT_B,
     reciprocalRankFusion,
     maximalMarginalRelevance,
     hybridRetrieve,
@@ -140,6 +148,7 @@ export type {
     Bm25Doc,
     Bm25Hit,
     Bm25Options,
+    Bm25Tokenizer,
     RankedList,
     FusedHit,
     MmrCandidate,
@@ -149,6 +158,226 @@ export type {
     HybridHit,
 } from './retrieval/index.js';
 
+// ── Telemetry (granular tracing + LLM-native metrics + cost-per-request) ──────
+export {
+    Tracer,
+    Span,
+    parseTraceparent,
+    formatTraceparent,
+    MetricsRegistry,
+    InstrumentedBridge,
+    InMemorySpanExporter,
+    ConsoleSpanExporter,
+    OtlpHttpSpanExporter,
+    toOtlpPayload,
+    toOtlpSpan,
+    percentile,
+    estimateCostUsd,
+    estimateTokens,
+    resolveRate,
+    DEFAULT_PRICE_BOOK,
+    ANTHROPIC_PRICE_BOOK,
+    EVERMIND_PRICE_BOOK,
+    DEFAULT_CACHE_READ_MULTIPLIER,
+    DEFAULT_CACHE_WRITE_MULTIPLIER,
+} from './telemetry/index.js';
+export type {
+    TracerOptions,
+    SpanOptions,
+    TraceContext,
+    SpanData,
+    SpanKind,
+    SpanStatus,
+    SpanEvent,
+    SpanExporter,
+    AttributeValue,
+    LlmUsage,
+    LlmMetricSnapshot,
+    KindMetricSnapshot,
+    MetricsRegistryOptions,
+    InstrumentedBridgeOptions,
+    ConsoleSpanExporterOptions,
+    OtlpHttpSpanExporterOptions,
+    ModelRate,
+    PriceBook,
+} from './telemetry/index.js';
+
+// ── Vector store (port + adapters + tenant/ACL scoping) ───────────────────────
+export {
+    MemoryVectorStore,
+    RestVectorStore,
+    getDialect,
+    listDialects,
+    registerDialect,
+    evermindDialect,
+    qdrantDialect,
+    pineconeDialect,
+    vertexAiDialect,
+    accessFilter,
+    combineFilters,
+    matchesFilter,
+    filterFields,
+    TENANT_FIELD,
+    ACL_FIELD,
+    SENSITIVITY_FIELD,
+    ACL_PUBLIC,
+} from './vectorstore/index.js';
+export type {
+    MemoryVectorStoreOptions,
+    RestVectorStoreOptions,
+    VectorDialect,
+    DialectContext,
+    TranslatedFilter,
+    AccessScope,
+    ComparisonOp,
+    MetadataFilter,
+    MetadataRecord,
+    MetadataValue,
+    UpsertResult,
+    VectorMatch,
+    VectorQuery,
+    VectorRecord,
+    VectorStore,
+} from './vectorstore/index.js';
+
+// ── Ingestion (structured + unstructured → vector store, incremental sync) ────
+export {
+    IngestionPipeline,
+    InMemoryIngestManifest,
+    ParserRegistry,
+    BUILTIN_PARSERS,
+    rowsParser,
+    csvParser,
+    jsonParser,
+    markdownParser,
+    htmlParser,
+    plainTextParser,
+    parseCsv,
+    serializeRow,
+    hashText,
+    mapWithConcurrency,
+    SOURCE_ID_FIELD,
+    CHUNK_INDEX_FIELD,
+    CONTENT_HASH_FIELD,
+    TITLE_FIELD,
+    URI_FIELD,
+    PARSER_FIELD,
+} from './ingest/index.js';
+export type {
+    IngestionPipelineOptions,
+    BatchEmbedder,
+    ChunkFingerprint,
+    DocumentParser,
+    IngestManifest,
+    IngestionReport,
+    ParsedSection,
+    SourceContent,
+    SourceDocument,
+    StructuredRow,
+    SyncStrategy,
+} from './ingest/index.js';
+
+// ── Enterprise RAG (hybrid, scoped, cited) ────────────────────────────────────
+export {
+    EnterpriseRetriever,
+    buildGroundedPrompt,
+    GROUNDED_SYSTEM_PROMPT,
+    NO_EVIDENCE_ANSWER,
+} from './rag/index.js';
+export type {
+    EnterpriseRetrieverOptions,
+    RetrieveOptions,
+    RetrievalResult,
+    RetrievalMode,
+    RetrievedPassage,
+    Reranker,
+} from './rag/index.js';
+
+// ── Multi-agent orchestration (graph + ReAct/reflection/delegation) ───────────
+export {
+    AgentGraph,
+    appendReducer,
+    mergeReducer,
+    sumReducer,
+    unionReducer,
+    InMemoryCheckpointer,
+    KeyValueCheckpointer,
+    createReactAgent,
+    createReflectionAgent,
+    createSupervisor,
+    asWorker,
+    parseReactTurn,
+    renderToolCatalog,
+    renderTranscript,
+    REACT_PROTOCOL,
+    DEFAULT_CRITIC_PROMPT,
+    FINISH,
+    START,
+    END,
+} from './orchestration/index.js';
+export type {
+    AgentGraphOptions,
+    RunOptions,
+    InMemoryCheckpointerOptions,
+    KeyValueCheckpointerOptions,
+    KeyValueLike,
+    AgentTool,
+    AgentTurn,
+    AgentTurnRole,
+    ParsedAction,
+    ReactAgentOptions,
+    ReactState,
+    ReflectionAgentOptions,
+    ReflectionState,
+    SupervisorOptions,
+    SupervisorState,
+    Worker,
+    ChannelReducer,
+    ChannelSpec,
+    ChannelSpecs,
+    Checkpoint,
+    Checkpointer,
+    EdgeCondition,
+    GraphEndReason,
+    GraphEvent,
+    GraphRunResult,
+    GraphState,
+    NodeContext,
+    NodeFn,
+} from './orchestration/index.js';
+
+// ── Evaluation (the conception → launch gate) ─────────────────────────────────
+export {
+    EvalHarness,
+    evaluateGate,
+    formatReport,
+    exactMatch,
+    substringChecks,
+    embeddingSimilarity,
+    retrievalRecall,
+    citationValidity,
+    llmJudge,
+    noError,
+    costBudget,
+    DEFAULT_RAG_GRADERS,
+    DEFAULT_JUDGE_RUBRIC,
+} from './eval/index.js';
+export type {
+    EvalHarnessOptions,
+    EvalCase,
+    EvalDataset,
+    EvalGate,
+    EvalOutput,
+    EvalReport,
+    EvalTarget,
+    CaseResult,
+    GateVerdict,
+    GradeResult,
+    Grader,
+    LlmJudgeOptions,
+    SimilarityGraderOptions,
+} from './eval/index.js';
+
 // ── Router ────────────────────────────────────────────────────────────────────
 export { InferenceRouter } from './router/InferenceRouter.js';
 export type {
@@ -166,6 +395,9 @@ export type {
     MemoryStoreOptions,
     RememberOptions,
     FactType,
+    RecallMethod,
+    ScoredMemory,
+    RankedRecall,
 } from './memory/MemoryStore.js';
 
 // ── Cognition (Evermind — Write-Through Cognition) ────────────────────────────