@remnic/core 1.1.12 → 1.1.13

package/src/enrichment/pipeline.ts

@@ -0,0 +1,197 @@
+/**
+ * Enrichment pipeline orchestrator (issue #365).
+ *
+ * For each entity, determines the importance tier, resolves the providers
+ * to run, executes them in sequence (respecting rate limits), tags
+ * candidates, and caps at `maxCandidatesPerEntity`.
+ *
+ * Accepted candidates are returned in each `EnrichmentResult` via the
+ * `acceptedCandidates` field so that callers can persist them.
+ */
+
+import type { LoggerBackend } from "../logger.js";
+import type { EnrichmentProviderRegistry } from "./provider-registry.js";
+import type {
+  EnrichmentCandidate,
+  EnrichmentPipelineConfig,
+  EnrichmentProvider,
+  EnrichmentResult,
+  EntityEnrichmentInput,
+} from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Rate-limit tracking
+// ---------------------------------------------------------------------------
+
+interface RateLimitBucket {
+  minuteCount: number;
+  minuteReset: number;
+  dayCount: number;
+  dayReset: number;
+}
+
+function isRateLimited(
+  provider: EnrichmentProvider,
+  config: EnrichmentPipelineConfig,
+  buckets: Map<string, RateLimitBucket>,
+): boolean {
+  const providerCfg = config.providers.find((p) => p.id === provider.id);
+  if (!providerCfg?.rateLimit) return false;
+
+  const now = Date.now();
+  let bucket = buckets.get(provider.id);
+  if (!bucket) {
+    bucket = {
+      minuteCount: 0,
+      minuteReset: now + 60_000,
+      dayCount: 0,
+      dayReset: now + 86_400_000,
+    };
+    buckets.set(provider.id, bucket);
+  }
+
+  // Reset windows if expired
+  if (now >= bucket.minuteReset) {
+    bucket.minuteCount = 0;
+    bucket.minuteReset = now + 60_000;
+  }
+  if (now >= bucket.dayReset) {
+    bucket.dayCount = 0;
+    bucket.dayReset = now + 86_400_000;
+  }
+
+  const { maxPerMinute, maxPerDay } = providerCfg.rateLimit;
+  return bucket.minuteCount >= maxPerMinute || bucket.dayCount >= maxPerDay;
+}
+
+function recordCall(
+  providerId: string,
+  buckets: Map<string, RateLimitBucket>,
+): void {
+  const bucket = buckets.get(providerId);
+  if (bucket) {
+    bucket.minuteCount += 1;
+    bucket.dayCount += 1;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline
+// ---------------------------------------------------------------------------
+
+export async function runEnrichmentPipeline(
+  entities: EntityEnrichmentInput[],
+  registry: EnrichmentProviderRegistry,
+  config: EnrichmentPipelineConfig,
+  log: LoggerBackend,
+): Promise<EnrichmentResult[]> {
+  if (!config.enabled) return [];
+  if (entities.length === 0) return [];
+
+  const rateBuckets = new Map<string, RateLimitBucket>();
+  const results: EnrichmentResult[] = [];
+
+  for (const entity of entities) {
+    const providers = registry.getForImportance(entity.importanceLevel, config);
+
+    for (const provider of providers) {
+      const start = Date.now();
+
+      // Check availability
+      let available: boolean;
+      try {
+        available = await provider.isAvailable();
+      } catch {
+        available = false;
+      }
+
+      if (!available) {
+        log.debug?.(
+          `enrichment: skipping provider ${provider.id} for ${entity.name} — unavailable`,
+        );
+        results.push({
+          entityName: entity.name,
+          provider: provider.id,
+          candidatesFound: 0,
+          candidatesAccepted: 0,
+          candidatesRejected: 0,
+          acceptedCandidates: [],
+          elapsed: Date.now() - start,
+        });
+        continue;
+      }
+
+      // Check rate limit
+      if (isRateLimited(provider, config, rateBuckets)) {
+        log.debug?.(
+          `enrichment: skipping provider ${provider.id} for ${entity.name} — rate limited`,
+        );
+        results.push({
+          entityName: entity.name,
+          provider: provider.id,
+          candidatesFound: 0,
+          candidatesAccepted: 0,
+          candidatesRejected: 0,
+          acceptedCandidates: [],
+          elapsed: Date.now() - start,
+        });
+        continue;
+      }
+
+      // Run provider.
+      // Count every attempt toward rate-limit buckets — including failures —
+      // because the provider may have consumed external quota before throwing
+      // (PR #425 review finding 2).
+      let candidates: EnrichmentCandidate[];
+      try {
+        candidates = await provider.enrich(entity);
+      } catch (err) {
+        recordCall(provider.id, rateBuckets);
+        log.error?.(
+          `enrichment: provider ${provider.id} failed for ${entity.name}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+        results.push({
+          entityName: entity.name,
+          provider: provider.id,
+          candidatesFound: 0,
+          candidatesAccepted: 0,
+          candidatesRejected: 0,
+          acceptedCandidates: [],
+          elapsed: Date.now() - start,
+        });
+        continue;
+      }
+      recordCall(provider.id, rateBuckets);
+
+      // Tag each candidate with provider id
+      for (const candidate of candidates) {
+        candidate.source = provider.id;
+      }
+
+      // Cap at maxCandidatesPerEntity.
+      // 0 means "accept none"; undefined/negative means "no cap".
+      const maxCandidates = config.maxCandidatesPerEntity;
+      let accepted: EnrichmentCandidate[];
+      if (maxCandidates === 0) {
+        accepted = [];
+      } else if (maxCandidates > 0 && candidates.length > maxCandidates) {
+        accepted = candidates.slice(0, maxCandidates);
+      } else {
+        accepted = candidates;
+      }
+      const rejected = candidates.length - accepted.length;
+
+      results.push({
+        entityName: entity.name,
+        provider: provider.id,
+        candidatesFound: candidates.length,
+        candidatesAccepted: accepted.length,
+        candidatesRejected: rejected,
+        acceptedCandidates: accepted,
+        elapsed: Date.now() - start,
+      });
+    }
+  }
+
+  return results;
+}

package/src/enrichment/provider-registry.ts

@@ -0,0 +1,85 @@
+/**
+ * Enrichment provider registry (issue #365).
+ *
+ * Central registry for enrichment providers. Providers register themselves
+ * at startup; the pipeline queries the registry to determine which providers
+ * to run for a given importance tier.
+ */
+
+import type { ImportanceLevel } from "../types.js";
+import type {
+  EnrichmentPipelineConfig,
+  EnrichmentProvider,
+} from "./types.js";
+
+export class EnrichmentProviderRegistry {
+  private readonly providers = new Map<string, EnrichmentProvider>();
+
+  /** Register a provider. Overwrites any existing provider with the same id. */
+  register(provider: EnrichmentProvider): void {
+    this.providers.set(provider.id, provider);
+  }
+
+  /** Look up a single provider by id. */
+  get(id: string): EnrichmentProvider | undefined {
+    return this.providers.get(id);
+  }
+
+  /**
+   * Return all registered providers whose id appears in the config's
+   * `providers` list with `enabled: true`.
+   */
+  listEnabled(config: EnrichmentPipelineConfig): EnrichmentProvider[] {
+    const enabledIds = new Set(
+      config.providers
+        .filter((p) => p.enabled)
+        .map((p) => p.id),
+    );
+    const result: EnrichmentProvider[] = [];
+    for (const [id, provider] of this.providers.entries()) {
+      if (enabledIds.has(id)) {
+        result.push(provider);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Return providers that should run for a given importance level.
+   * Providers are resolved from `config.importanceThresholds[level]` and
+   * filtered to only those that are both registered and enabled.
+   */
+  getForImportance(
+    level: ImportanceLevel,
+    config: EnrichmentPipelineConfig,
+  ): EnrichmentProvider[] {
+    // "trivial" entities never get enrichment providers
+    if (level === "trivial") return [];
+
+    const thresholds = config.importanceThresholds;
+    const providerIds: string[] =
+      level === "critical"
+        ? thresholds.critical
+        : level === "high"
+          ? thresholds.high
+          : level === "normal"
+            ? thresholds.normal
+            : thresholds.low;
+
+    const enabledIds = new Set(
+      config.providers
+        .filter((p) => p.enabled)
+        .map((p) => p.id),
+    );
+
+    const result: EnrichmentProvider[] = [];
+    for (const id of providerIds) {
+      if (!enabledIds.has(id)) continue;
+      const provider = this.providers.get(id);
+      if (provider) {
+        result.push(provider);
+      }
+    }
+    return result;
+  }
+}

package/src/enrichment/types.ts

@@ -0,0 +1,100 @@
+/**
+ * Enrichment pipeline types (issue #365).
+ *
+ * Defines the provider interface, candidate shape, pipeline config,
+ * and result types for the external enrichment subsystem.
+ */
+
+import type { ImportanceLevel, MemoryCategory } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Provider config & interface
+// ---------------------------------------------------------------------------
+
+export type EnrichmentCostTier = "free" | "cheap" | "expensive";
+
+export interface EnrichmentProviderConfig {
+  id: string;
+  enabled: boolean;
+  costTier: EnrichmentCostTier;
+  rateLimit?: { maxPerMinute: number; maxPerDay: number };
+}
+
+export interface EnrichmentCandidate {
+  text: string;
+  source: string;
+  sourceUrl?: string;
+  confidence: number;
+  category: MemoryCategory;
+  tags?: string[];
+}
+
+export interface EnrichmentProvider {
+  readonly id: string;
+  readonly costTier: EnrichmentCostTier;
+  enrich(entity: EntityEnrichmentInput): Promise<EnrichmentCandidate[]>;
+  isAvailable(): Promise<boolean>;
+}
+
+// ---------------------------------------------------------------------------
+// Entity enrichment input
+// ---------------------------------------------------------------------------
+
+export interface EntityEnrichmentInput {
+  name: string;
+  type: string;
+  knownFacts: string[];
+  importanceLevel: ImportanceLevel;
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline result
+// ---------------------------------------------------------------------------
+
+export interface EnrichmentResult {
+  entityName: string;
+  provider: string;
+  candidatesFound: number;
+  candidatesAccepted: number;
+  candidatesRejected: number;
+  acceptedCandidates: EnrichmentCandidate[];
+  elapsed: number;
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline config
+// ---------------------------------------------------------------------------
+
+export interface EnrichmentPipelineConfig {
+  enabled: boolean;
+  providers: EnrichmentProviderConfig[];
+  importanceThresholds: {
+    critical: string[];
+    high: string[];
+    normal: string[];
+    low: string[];
+  };
+  maxCandidatesPerEntity: number;
+  autoEnrichOnCreate: boolean;
+  scheduleIntervalMs: number;
+}
+
+/**
+ * Build a default (disabled) pipeline config. Every consumer that needs a
+ * config object should call this rather than duplicating the defaults.
+ */
+export function defaultEnrichmentPipelineConfig(): EnrichmentPipelineConfig {
+  return {
+    enabled: false,
+    providers: [],
+    importanceThresholds: {
+      critical: [],
+      high: [],
+      normal: [],
+      low: [],
+    },
+    maxCandidatesPerEntity: 20,
+    autoEnrichOnCreate: false,
+    scheduleIntervalMs: 3_600_000,
+  };
+}

package/src/enrichment/web-search-provider.ts

@@ -0,0 +1,63 @@
+/**
+ * Web search enrichment provider stub (issue #365).
+ *
+ * A basic provider backed by web search. Since this is opt-in and we do not
+ * want to hard-code an API key, the provider accepts an optional `searchFn`
+ * injection point. When no search function is configured it returns empty
+ * results, making it safe to register unconditionally.
+ */
+
+import type {
+  EnrichmentCandidate,
+  EnrichmentCostTier,
+  EnrichmentProvider,
+  EntityEnrichmentInput,
+} from "./types.js";
+
+export type WebSearchFn = (query: string) => Promise<string[]>;
+
+export interface WebSearchProviderOptions {
+  /**
+   * Injected search function. Each returned string is treated as a raw
+   * snippet. When `undefined` the provider returns empty results.
+   */
+  searchFn?: WebSearchFn;
+}
+
+export class WebSearchProvider implements EnrichmentProvider {
+  readonly id = "web-search";
+  readonly costTier: EnrichmentCostTier = "cheap";
+
+  private readonly searchFn: WebSearchFn | undefined;
+
+  constructor(options: WebSearchProviderOptions = {}) {
+    this.searchFn = options.searchFn;
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return this.searchFn !== undefined;
+  }
+
+  async enrich(entity: EntityEnrichmentInput): Promise<EnrichmentCandidate[]> {
+    if (!this.searchFn) return [];
+
+    const query = `${entity.name} ${entity.type}`;
+    let snippets: string[];
+    try {
+      snippets = await this.searchFn(query);
+    } catch {
+      return [];
+    }
+
+    return snippets
+      .filter((s) => typeof s === "string" && s.trim().length > 0)
+      .map((snippet) => ({
+        text: snippet.trim(),
+        source: this.id,
+        sourceUrl: undefined,
+        confidence: 0.5,
+        category: "fact" as const,
+        tags: ["web-search"],
+      }));
+  }
+}