atlas-mcp 0.1.0

package/src/extraction-context.js

@@ -0,0 +1,31 @@
+export const EXTRACTION_CONTEXT_LIMIT = 5;
+export const EXTRACTION_CONTEXT_THRESHOLD = 0.7;
+
+export async function retrieveExtractionContext(
+  text,
+  { searchMemoryVectors, getEntitiesForMemory },
+) {
+  const hits = await searchMemoryVectors(text, {
+    limit: EXTRACTION_CONTEXT_LIMIT,
+  });
+
+  const entities = hits
+    .filter(({ score }) => Number(score) >= EXTRACTION_CONTEXT_THRESHOLD)
+    .slice(0, EXTRACTION_CONTEXT_LIMIT)
+    .flatMap(({ id }) => getEntitiesForMemory?.(id) || [])
+    .map((entity) => ({
+      canonicalName: entity.canonical_name || entity.canonicalName,
+      aliases: [entity.mention].filter(Boolean),
+      kind: entity.kind,
+    }));
+  const unique = new Map();
+  for (const entity of entities) {
+    if (!entity.canonicalName || !entity.kind) continue;
+    const key = `${entity.kind}:${entity.canonicalName.toLocaleLowerCase()}`;
+    const current = unique.get(key);
+    unique.set(key, current
+      ? { ...current, aliases: [...new Set([...current.aliases, ...entity.aliases])] }
+      : entity);
+  }
+  return { entities: [...unique.values()] };
+}

package/src/ingestion-service.js

@@ -0,0 +1,387 @@
+import { randomUUID } from "node:crypto";
+import { retrieveExtractionContext } from "./extraction-context.js";
+import {
+  COGNITIVE_ANNOTATION_SCHEMA_VERSION,
+  SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+} from "./schemas.js";
+
+const DEFAULT_METADATA = Object.freeze({ confidence: 0.6, tags: [] });
+
+export function createIngestionService(dependencies) {
+  for (const name of [
+    "createMemorySource", "updateMemorySourceStatus", "extractAtomicMemories",
+    "storeMemory", "updateMemoryGraph", "getMemory", "linkSourceMemory",
+    "enqueueAnnotationJob", "enqueueVectorIndexJob", "withTransaction",
+  ]) {
+    if (typeof dependencies[name] !== "function") {
+      throw new Error(`Ingestion service is missing required dependency: ${name}`);
+    }
+  }
+
+  async function ingest({
+    text,
+    ingestionDate = new Date().toISOString(),
+    source = "ui",
+    metadata = {},
+    sourceId = randomUUID(),
+  }) {
+    const existing = dependencies.getMemorySource?.(sourceId);
+    if (existing) {
+      if (existing.text !== text) {
+        throw new Error(`Source ${sourceId} already exists with different text`);
+      }
+      if (existing.extraction_status === "completed") {
+        return completedSourceResult(existing);
+      }
+    } else {
+      dependencies.createMemorySource({
+        id: sourceId,
+        text,
+        source,
+        ingestionDate,
+        metadata,
+      });
+    }
+    return processSource({ sourceId, text, ingestionDate, source, metadata });
+  }
+
+  async function enqueue({
+    text,
+    ingestionDate = new Date().toISOString(),
+    source = "ui",
+    metadata = {},
+    sourceId = randomUUID(),
+  }) {
+    if (typeof dependencies.enqueueIngestionJob !== "function") {
+      throw new Error("Ingestion service is missing required dependency: enqueueIngestionJob");
+    }
+    const existing = dependencies.getMemorySource?.(sourceId);
+    if (existing) {
+      if (existing.text !== text) {
+        throw new Error(`Source ${sourceId} already exists with different text`);
+      }
+      if (existing.extraction_status === "completed") {
+        return completedSourceResult(existing);
+      }
+    } else {
+      dependencies.createMemorySource({
+        id: sourceId,
+        text,
+        source,
+        ingestionDate,
+        metadata,
+      });
+    }
+    dependencies.enqueueIngestionJob({ sourceId });
+    return { sourceId, status: "queued", memories: [] };
+  }
+
+  async function runIngestion(sourceId, { metadata = {} } = {}) {
+    const sourceRecord = dependencies.getMemorySource?.(sourceId);
+    if (!sourceRecord) throw new Error(`Source not found: ${sourceId}`);
+    return processSource({
+      sourceId,
+      text: sourceRecord.text,
+      ingestionDate: sourceRecord.ingestion_date,
+      source: sourceRecord.source,
+      metadata: { ...sourceRecord.metadata_json, ...metadata },
+    });
+  }
+
+  async function reprocess(sourceId, { metadata = {} } = {}) {
+    const sourceRecord = dependencies.getMemorySource(sourceId, {
+      includeRevisions: true,
+    });
+    if (!sourceRecord) throw new Error(`Source not found: ${sourceId}`);
+    const revision = sourceRecord.revisions?.[0];
+    return processSource({
+      sourceId,
+      sourceRevisionId: revision?.id ?? null,
+      text: revision?.text ?? sourceRecord.text,
+      ingestionDate: sourceRecord.ingestion_date,
+      source: sourceRecord.source,
+      metadata: { ...sourceRecord.metadata_json, ...metadata },
+    });
+  }
+
+  async function processSource({
+    sourceId,
+    sourceRevisionId = null,
+    text,
+    ingestionDate,
+    source,
+    metadata,
+  }) {
+    const model = await resolveModel(dependencies);
+    dependencies.updateMemorySourceStatus(sourceId, "processing", {
+      incrementAttempts: true,
+      model,
+      schemaVersion: SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+    });
+
+    try {
+      const canonicalizationContext = await optionalCanonicalizationContext(
+        text,
+        dependencies,
+      );
+      const semanticExtraction = await dependencies.extractAtomicMemories(
+        text,
+        ingestionDate,
+        canonicalizationContext,
+      );
+
+      const prepared = [];
+      for (const atom of semanticExtraction.memories) {
+        prepared.push(await prepareAtom({
+          atom,
+          source,
+          ingestionDate,
+          metadata,
+          model,
+        }));
+      }
+
+      const outcomes = dependencies.withTransaction(() => {
+        const persisted = prepared.map((item) => persistAtom({
+          ...item,
+          sourceId,
+          sourceRevisionId,
+          source,
+          ingestionDate,
+          model,
+        }));
+        dependencies.updateMemorySourceStatus(sourceId, "completed", {
+          model,
+          schemaVersion: SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+        });
+        return persisted;
+      });
+
+      return {
+        sourceId,
+        status: "completed",
+        memories: outcomes,
+        ...(outcomes.length === 0 ? { reason: "no_durable_memory" } : {}),
+      };
+    } catch (error) {
+      dependencies.updateMemorySourceStatus(sourceId, "failed", {
+        error: error.message,
+        model,
+        schemaVersion: SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+      });
+      error.sourceId = sourceId;
+      error.code ||= "INGESTION_FAILED";
+      throw error;
+    }
+  }
+
+  async function prepareAtom({ atom, source, ingestionDate, metadata, model }) {
+    const candidates = await writeCandidates(atom.text, dependencies);
+    let decision = createDecision("No similar stored memory was found.", 1);
+    if (candidates.available && candidates.items.length > 0) {
+      try {
+        decision = await dependencies.decideMemoryWrite(
+          { text: atom.text, summary: atom.summary, ...atomMetadata(atom, metadata) },
+          candidates.items,
+        );
+      } catch {
+        decision = createDecision("Memory matching was inconclusive.");
+      }
+    } else if (!candidates.available) {
+      decision = createDecision("Memory candidate search was unavailable.");
+    }
+
+    if (decision.action !== "create") {
+      const valid = decision.confidence >= 0.85
+        && candidates.items.some(({ id }) => id === decision.matchedMemoryId);
+      if (!valid) decision = createDecision("The possible memory match was uncertain.");
+    }
+
+    let storedAtom = atom;
+    const replacementText = decision.replacementText?.trim();
+    if (decision.action === "update" && replacementText && replacementText !== atom.text) {
+      const replacement = await dependencies.extractAtomicMemories(
+        replacementText,
+        ingestionDate,
+        await optionalCanonicalizationContext(replacementText, dependencies),
+      );
+      if (replacement.memories.length !== 1) {
+        throw new Error("Memory update replacement must resolve to exactly one atomic memory");
+      }
+      storedAtom = replacement.memories[0];
+    }
+
+    return {
+      atom: storedAtom,
+      sourceEvidence: atom.evidenceSpans,
+      decision,
+      memoryId: decision.action === "create"
+        ? dependencies.createMemoryId?.() ?? randomUUID()
+        : decision.matchedMemoryId,
+      metadata: atomMetadata(storedAtom, metadata),
+      source,
+      ingestionDate,
+      model,
+    };
+  }
+
+  function persistAtom({
+    atom, sourceEvidence, decision, memoryId, metadata, sourceId,
+    sourceRevisionId, source, ingestionDate, model,
+  }) {
+    let memory;
+    if (decision.action === "unchanged") {
+      memory = dependencies.getMemory(memoryId);
+      if (!memory) throw new Error(`Matched memory not found: ${memoryId}`);
+    } else if (decision.action === "update") {
+      memory = dependencies.updateMemoryGraph({
+        memoryId,
+        rawText: atom.text,
+        ingestionDate,
+        extraction: atom,
+        model,
+        metadata,
+        schemaVersion: SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+      }).memory;
+    } else {
+      memory = dependencies.storeMemory(
+        memoryId,
+        atom.text,
+        ingestionDate,
+        atom,
+        model,
+        source,
+        metadata,
+      );
+    }
+
+    const action = normalizeAction(decision.action);
+    dependencies.linkSourceMemory({
+      sourceId,
+      sourceRevisionId,
+      memoryId,
+      action,
+      evidence: sourceEvidence,
+      model,
+      schemaVersion: SEMANTIC_EXTRACTION_SCHEMA_VERSION,
+      confidence: decision.confidence,
+      reason: decision.reason,
+    });
+    if (action !== "unchanged") {
+      dependencies.enqueueAnnotationJob({
+        memoryId,
+        sourceId,
+        model,
+        schemaVersion: COGNITIVE_ANNOTATION_SCHEMA_VERSION,
+      });
+      dependencies.enqueueVectorIndexJob({ memoryId, sourceId });
+    }
+
+    const serialized = dependencies.serializeMemory?.(memory) ?? memory;
+    return {
+      action,
+      memory: serialized,
+      matchedMemoryId: action === "created" ? null : decision.matchedMemoryId,
+      confidence: decision.confidence,
+      reason: decision.reason,
+      evidenceSpans: sourceEvidence,
+      annotationStatus: action === "unchanged"
+        ? normalizeStatus(dependencies.getAnnotationStatus?.(memoryId), "completed")
+        : "pending",
+      indexStatus: action === "unchanged"
+        ? normalizeStatus(dependencies.getVectorIndexStatus?.(memoryId), "completed")
+        : "pending",
+    };
+  }
+
+  function completedSourceResult(sourceRecord) {
+    const links = dependencies.getSourceMemoryLinks?.(sourceRecord.id) || [];
+    return {
+      sourceId: sourceRecord.id,
+      status: "completed",
+      memories: links.map((link) => ({
+        action: link.action,
+        memory: dependencies.serializeMemory?.(dependencies.getMemory(link.memory_id))
+          ?? dependencies.getMemory(link.memory_id),
+        matchedMemoryId: link.action === "created" ? null : link.memory_id,
+        confidence: link.decision_confidence,
+        reason: link.decision_reason,
+        evidenceSpans: link.evidence_json,
+        annotationStatus: normalizeStatus(
+          dependencies.getAnnotationStatus?.(link.memory_id),
+          "completed",
+        ),
+        indexStatus: normalizeStatus(
+          dependencies.getVectorIndexStatus?.(link.memory_id),
+          "completed",
+        ),
+      })),
+    };
+  }
+
+  return { ingest, enqueue, runIngestion, reprocess };
+}
+
+async function optionalCanonicalizationContext(text, dependencies) {
+  try {
+    return await retrieveExtractionContext(text, dependencies);
+  } catch {
+    return { entities: [] };
+  }
+}
+
+async function writeCandidates(text, dependencies) {
+  try {
+    const hits = await dependencies.searchMemoryVectors(text, { limit: 5 });
+    return {
+      available: true,
+      items: hits.map(({ id }) => serializeCandidate(id, dependencies)).filter(Boolean),
+    };
+  } catch {
+    return { available: false, items: [] };
+  }
+}
+
+async function resolveModel(dependencies) {
+  return dependencies.model || dependencies.getModel?.() || "unknown";
+}
+
+function atomMetadata(atom, metadata) {
+  const extractedType = dominantType(atom.types);
+  return {
+    ...DEFAULT_METADATA,
+    type: metadata.forceType && metadata.type
+      ? metadata.type
+      : extractedType || metadata.type || "fact",
+    title: atom.summary || atom.text.slice(0, 50),
+    confidence: atom.durability?.confidence ?? DEFAULT_METADATA.confidence,
+    tags: Array.isArray(metadata.tags) ? metadata.tags : [],
+  };
+}
+
+function dominantType(types = []) {
+  return [...types].sort((left, right) =>
+    right.weight - left.weight || left.type.localeCompare(right.type))[0]?.type || null;
+}
+
+function serializeCandidate(id, dependencies) {
+  const memory = dependencies.getMemory(id);
+  if (!memory) return null;
+  const extraction = dependencies.getLatestExtraction?.(id)?.extraction_json ?? null;
+  return { id: memory.id, text: memory.raw_text, summary: memory.summary,
+    type: memory.type, title: memory.title, tags: memory.tags, extraction };
+}
+
+function createDecision(reason, confidence = 0) {
+  return { action: "create", matchedMemoryId: null, confidence, reason, replacementText: "" };
+}
+
+function normalizeAction(action) {
+  return action === "create" ? "created" : action === "update" ? "updated" : "unchanged";
+}
+
+function normalizeStatus(status, fallback) {
+  return ["pending", "processing", "completed", "failed"].includes(status)
+    ? status
+    : fallback;
+}

package/src/ingestion-worker.js

@@ -0,0 +1,225 @@
+import { calculateRetryDelay } from "./cognitive-worker.js";
+
+const DEFAULT_POLL_INTERVAL_MS = 1_000;
+const DEFAULT_STALE_AFTER_MS = 5 * 60_000;
+const DEFAULT_BASE_RETRY_MS = 1_000;
+const DEFAULT_MAX_RETRY_MS = 60_000;
+const DEFAULT_MAX_ATTEMPTS = 5;
+
+function requiredFunction(value, name) {
+  if (typeof value !== "function") {
+    throw new TypeError(`ingestion worker requires ${name}()`);
+  }
+  return value;
+}
+
+function optionalFunction(value) {
+  return typeof value === "function" ? value : null;
+}
+
+function errorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+function jobId(job) {
+  return job.id ?? job.jobId ?? job.job_id;
+}
+
+function sourceId(job) {
+  return job.sourceId ?? job.source_id ?? job.source?.id;
+}
+
+function attemptCount(job) {
+  const value = Number(job.attempts ?? job.attemptCount ?? job.attempt_count);
+  return Number.isInteger(value) && value > 0 ? value : 1;
+}
+
+function maximumAttempts(job, fallback) {
+  const value = Number(job.maxAttempts ?? job.max_attempts ?? job.maximumAttempts);
+  return Number.isInteger(value) && value > 0 ? Math.min(value, fallback) : fallback;
+}
+
+function validateOptions(options) {
+  for (const [name, value] of [
+    ["pollIntervalMs", options.pollIntervalMs],
+    ["staleAfterMs", options.staleAfterMs],
+    ["baseRetryMs", options.baseRetryMs],
+    ["maxRetryMs", options.maxRetryMs],
+  ]) {
+    if (!Number.isFinite(value) || value < 0) {
+      throw new RangeError(`${name} must be a non-negative finite number`);
+    }
+  }
+  if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
+    throw new RangeError("maxAttempts must be a positive integer");
+  }
+  if (options.maxRetryMs < options.baseRetryMs) {
+    throw new RangeError("maxRetryMs must be at least baseRetryMs");
+  }
+}
+
+export function createIngestionWorker({
+  db,
+  ingestionService,
+  now = () => new Date(),
+  setTimer = setTimeout,
+  clearTimer = clearTimeout,
+  pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+  staleAfterMs = DEFAULT_STALE_AFTER_MS,
+  baseRetryMs = DEFAULT_BASE_RETRY_MS,
+  maxRetryMs = DEFAULT_MAX_RETRY_MS,
+  maxAttempts = DEFAULT_MAX_ATTEMPTS,
+  onError = () => {},
+} = {}) {
+  if (!db || typeof db !== "object") {
+    throw new TypeError("ingestion worker requires a db adapter");
+  }
+  if (!ingestionService || typeof ingestionService.runIngestion !== "function") {
+    throw new TypeError("ingestion worker requires ingestionService.runIngestion()");
+  }
+
+  const options = { pollIntervalMs, staleAfterMs, baseRetryMs, maxRetryMs, maxAttempts };
+  validateOptions(options);
+
+  const claimJob = requiredFunction(db.claimIngestionJob, "db.claimIngestionJob");
+  const recoverJobs = requiredFunction(db.recoverIngestionJobs, "db.recoverIngestionJobs");
+  const completeJob = requiredFunction(db.completeIngestionJob, "db.completeIngestionJob");
+  const retryJob = requiredFunction(db.retryIngestionJob, "db.retryIngestionJob");
+  const failJob = optionalFunction(db.failIngestionJob);
+
+  let timer = null;
+  let resolvePoll = null;
+  let loopPromise = null;
+  let stopped = true;
+
+  const currentDate = () => {
+    const value = now();
+    const date = value instanceof Date ? value : new Date(value);
+    if (Number.isNaN(date.getTime())) {
+      throw new TypeError("now() must return a valid Date or date value");
+    }
+    return date;
+  };
+
+  const reschedule = async (job, error) => {
+    const attempts = attemptCount(job);
+    const attemptLimit = maximumAttempts(job, maxAttempts);
+    const id = jobId(job);
+    const message = errorMessage(error);
+    const failedAt = currentDate();
+
+    if (attempts >= attemptLimit && failJob) {
+      await failJob.call(db, id, {
+        error: message,
+        attempts,
+        failedAt: failedAt.toISOString(),
+      });
+      return { status: "failed", job, error };
+    }
+
+    const delayMs = calculateRetryDelay(attempts, { baseRetryMs, maxRetryMs });
+    const retryAt = new Date(failedAt.getTime() + delayMs);
+    await retryJob.call(db, {
+      jobId: id,
+      error: message,
+      attempts,
+      retryAt: retryAt.toISOString(),
+      terminal: attempts >= attemptLimit,
+      updatedAt: failedAt.toISOString(),
+    });
+    return {
+      status: attempts >= attemptLimit ? "failed" : "retrying",
+      job,
+      error,
+      retryAt,
+      delayMs,
+    };
+  };
+
+  const runOnce = async () => {
+    const claimedAt = currentDate();
+    const job = await claimJob.call(db, { now: claimedAt.toISOString() });
+    if (!job) return { status: "idle" };
+
+    try {
+      const id = sourceId(job);
+      if (!id) throw new Error(`ingestion job ${jobId(job)} has no source id`);
+      const result = await ingestionService.runIngestion(id);
+      await completeJob.call(db, {
+        jobId: jobId(job),
+        completedAt: currentDate().toISOString(),
+      });
+      return { status: "completed", job, result };
+    } catch (error) {
+      return reschedule(job, error);
+    }
+  };
+
+  const waitForPoll = () =>
+    new Promise((resolve) => {
+      resolvePoll = resolve;
+      timer = setTimer(() => {
+        timer = null;
+        resolvePoll = null;
+        resolve();
+      }, pollIntervalMs);
+    });
+
+  const run = async ({ signal, recover = true } = {}) => {
+    stopped = false;
+    if (recover) {
+      const recoveredAt = currentDate();
+      await recoverJobs.call(db, {
+        now: recoveredAt.toISOString(),
+        retryAt: recoveredAt.toISOString(),
+        staleBefore: new Date(recoveredAt.getTime() - staleAfterMs).toISOString(),
+      });
+    }
+
+    while (!stopped && !signal?.aborted) {
+      const result = await runOnce();
+      if (result.status === "idle") await waitForPoll();
+    }
+  };
+
+  const start = () => {
+    if (loopPromise) return loopPromise;
+    stopped = false;
+    loopPromise = run()
+      .catch((error) => {
+        onError(error);
+        throw error;
+      })
+      .finally(() => {
+        loopPromise = null;
+        stopped = true;
+      });
+    return loopPromise;
+  };
+
+  const stop = async () => {
+    stopped = true;
+    if (timer !== null) {
+      clearTimer(timer);
+      timer = null;
+      resolvePoll?.();
+      resolvePoll = null;
+    }
+    await loopPromise;
+  };
+
+  return {
+    start,
+    run,
+    runOnce,
+    stop,
+    get running() {
+      return loopPromise !== null;
+    },
+  };
+}
+
+export async function runIngestionWorkerCycle(dependencies) {
+  return createIngestionWorker(dependencies).runOnce();
+}

package/src/llm-config.js

@@ -0,0 +1,31 @@
+import createLogger from "./logger.js";
+
+const log = createLogger("llm-config");
+
+const PROVIDERS = {
+  tokenrouter: {
+    baseUrl: "https://api.tokenrouter.com/v1/chat/completions",
+    model: "MiniMax-M3",
+    apiKeyEnv: "TOKENROUTER_API_KEY",
+  },
+  openrouter: {
+    baseUrl: "https://openrouter.ai/api/v1/chat/completions",
+    model: "deepseek/deepseek-v4-flash",
+    apiKeyEnv: "OPENROUTER_API_KEY",
+  },
+};
+
+const activeProvider = process.env.LLM_PROVIDER || "tokenrouter";
+
+const config = PROVIDERS[activeProvider];
+
+if (!config) {
+  throw new Error(`Unknown LLM_PROVIDER "${activeProvider}". Valid: ${Object.keys(PROVIDERS).join(", ")}`);
+}
+
+export const baseUrl = config.baseUrl;
+export const model = process.env.LLM_MODEL || config.model;
+export const apiKeyEnv = config.apiKeyEnv;
+export const providerName = activeProvider;
+
+log.info("provider configured", { provider: activeProvider, model, baseUrl: config.baseUrl });