fuli-context 0.2.0

package/src/http/response.js

@@ -0,0 +1,29 @@
+// Keep local API request buffering at or below a fixed 64 KiB budget.
+export const MAX_JSON_BODY_BYTES = 64 * 1024;
+
+export class JsonBodyTooLargeError extends Error {
+  constructor() {
+    super(`JSON request body exceeds ${MAX_JSON_BODY_BYTES} bytes`);
+    this.name = 'JsonBodyTooLargeError';
+  }
+}
+
+export async function readJson(request) {
+  const chunks = [];
+  let receivedBytes = 0;
+  for await (const chunk of request) {
+    receivedBytes += chunk.length;
+    if (receivedBytes > MAX_JSON_BODY_BYTES) {
+      chunks.length = 0;
+      throw new JsonBodyTooLargeError();
+    }
+    chunks.push(chunk);
+  }
+  const raw = Buffer.concat(chunks).toString('utf8');
+  return raw ? JSON.parse(raw) : {};
+}
+
+export function sendJson(response, status, body) {
+  response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
+  response.end(JSON.stringify(body));
+}

package/src/http/static-handler.js

@@ -0,0 +1,55 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { extname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const PROJECT_ROOT = resolve(fileURLToPath(new URL('../..', import.meta.url)));
+export const WEB_ROOT = join(PROJECT_ROOT, 'dist', 'web');
+const MIME_TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.map': 'application/json; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml',
+  '.webp': 'image/webp',
+  '.woff2': 'font/woff2'
+};
+
+export function serveStatic(pathname, response) {
+  if (pathname === '/favicon.ico') {
+    response.writeHead(204);
+    response.end();
+    return;
+  }
+
+  const relativePath = pathname === '/' ? 'index.html' : pathname.slice(1);
+  const requestedFile = normalize(join(WEB_ROOT, relativePath));
+  const pathFromRoot = relative(WEB_ROOT, requestedFile);
+  if (pathFromRoot === '..' || pathFromRoot.startsWith(`..${sep}`) ||
+      isAbsolute(pathFromRoot)) {
+    response.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' });
+    response.end('Not found');
+    return;
+  }
+
+  const filePath = existsSync(requestedFile)
+    ? requestedFile
+    : shouldServeApplication(pathname)
+      ? join(WEB_ROOT, 'index.html')
+      : null;
+  if (!filePath || !existsSync(filePath)) {
+    response.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' });
+    response.end('Not found');
+    return;
+  }
+
+  response.writeHead(200, {
+    'content-type': MIME_TYPES[extname(filePath)] ?? 'application/octet-stream'
+  });
+  response.end(readFileSync(filePath));
+}
+
+function shouldServeApplication(pathname) {
+  return extname(pathname) === '' && !pathname.startsWith('/api/');
+}

package/src/ingestion.js

@@ -0,0 +1,161 @@
+import { classifyEpisode } from './classifier.js';
+import { extractFactSpecs } from './extractor.js';
+import { FactService } from './facts/fact-service.js';
+import { LensService } from './lens/lens-service.js';
+import { ApplicationError, ApplicationErrorCode } from './app/application-error.js';
+import { CandidateStatus, FactScope, PublishRoute, SpaceKind } from './models.js';
+import { detectSensitiveContent } from './security/sensitive-content.js';
+import { assertSafeSourceMetadata } from './security/source-metadata-policy.js';
+import { PublicationService } from './publication/publication-service.js';
+
+export class IngestionService {
+  constructor(store, {
+    facts = new FactService(store),
+    lens = new LensService(store),
+    publication = new PublicationService(store)
+  } = {}) {
+    this.store = store;
+    this.facts = facts;
+    this.lens = lens;
+    this.publication = publication;
+  }
+
+  remember(input) {
+    this.#assertSafeInput(input);
+    return this.store.transaction(
+      () => this.rememberInCurrentTransaction(input),
+      { mode: 'immediate' }
+    );
+  }
+
+  rememberInCurrentTransaction({
+    personalSpaceId,
+    sourceKind,
+    body,
+    targetSpaceId = null,
+    sourceUri = null
+  }) {
+    this.#assertSafeInput({ body, sourceKind, sourceUri });
+    const route = classifyEpisode({ sourceKind, body });
+
+    if (route === PublishRoute.PERSONAL) {
+      const result = this.lens.rememberUserFactsFromSourceInCurrentTransaction({
+        personalSpaceId,
+        subject: this.#spaceName(personalSpaceId),
+        specs: extractFactSpecs(this.#spaceName(personalSpaceId), body),
+        sourceText: body,
+        sourceKind,
+        sourceUri
+      });
+      return { route, ...result };
+    }
+
+    const personalEpisode = this.store.addEpisode(personalSpaceId, sourceKind, body, sourceUri);
+
+    if (route === PublishRoute.PUBLIC && targetSpaceId) {
+      const targetEpisode = this.store.addEpisode(targetSpaceId, sourceKind, body, sourceUri);
+      const facts = this.facts.writeSpecs({
+        spaceId: targetSpaceId,
+        subject: this.#spaceName(targetSpaceId),
+        episodeId: targetEpisode.id,
+        specs: extractFactSpecs(this.#spaceName(targetSpaceId), body),
+        scope: FactScope.PUBLIC
+      });
+      const publication = facts.length === 0
+        ? null
+        : this.publication.prepareInCurrentTransaction({
+          spaceId: targetSpaceId,
+          episode: targetEpisode,
+          facts
+        });
+      return { route, episode: targetEpisode, publication };
+    }
+
+    const candidate = this.store.addCandidate({
+      personalSpaceId,
+      targetSpaceId,
+      episodeId: personalEpisode.id,
+      reason: 'not safe enough to publish automatically',
+      status: CandidateStatus.PENDING
+    });
+    return { route: PublishRoute.CANDIDATE, episode: personalEpisode, candidate };
+  }
+
+  publishConfirmed(input) {
+    this.#assertSafeInput(input);
+    return this.store.transaction(
+      () => this.publishConfirmedInCurrentTransaction(input),
+      { mode: 'immediate' }
+    );
+  }
+
+  publishConfirmedInCurrentTransaction({ targetSpaceId, sourceKind, body, sourceUri = null }) {
+    this.#assertSafeInput({ body, sourceKind, sourceUri });
+    const targetEpisode = this.store.addEpisode(targetSpaceId, sourceKind, body, sourceUri);
+    const result = this.confirmEpisodeInCurrentTransaction({
+      spaceId: targetSpaceId,
+      episode: targetEpisode
+    });
+    return result;
+  }
+
+  confirmEpisode(input) {
+    this.#assertSafeEpisode(input.episode);
+    return this.store.transaction(
+      () => this.confirmEpisodeInCurrentTransaction(input),
+      { mode: 'immediate' }
+    );
+  }
+
+  confirmEpisodeInCurrentTransaction({ spaceId, episode }) {
+    this.#assertSafeEpisode(episode);
+    const subject = this.#spaceName(spaceId);
+    const specs = extractFactSpecs(subject, episode.body);
+    if (this.store.getSpace(spaceId)?.kind === SpaceKind.PERSONAL) {
+      return this.lens.writeUserFactsForEpisodeInCurrentTransaction({
+        personalSpaceId: spaceId,
+        subject,
+        specs,
+        episode
+      });
+    }
+    const facts = this.facts.writeSpecs({
+      spaceId,
+      subject,
+      episodeId: episode.id,
+      specs,
+      scope: FactScope.PUBLIC
+    });
+    const publication = facts.length === 0
+      ? null
+      : this.publication.prepareInCurrentTransaction({ spaceId, episode, facts });
+    return { episode, publication };
+  }
+
+  #spaceName(spaceId) {
+    return this.store.getSpace(spaceId)?.name ?? spaceId;
+  }
+
+  #assertSafeInput({ body, sourceKind, sourceUri }) {
+    this.#assertSafeBody(body);
+    assertSafeSourceMetadata(
+      sourceKind,
+      sourceUri,
+      'Sensitive source metadata is not allowed in ingestion'
+    );
+  }
+
+  #assertSafeEpisode(episode) {
+    this.#assertSafeInput(episode);
+  }
+
+  #assertSafeBody(body) {
+    if (detectSensitiveContent(body).restricted) {
+      throw new ApplicationError(
+        ApplicationErrorCode.VALIDATION,
+        'Sensitive content is not allowed in ingestion'
+      );
+    }
+  }
+
+}

package/src/lens/lens-correction-service.js

@@ -0,0 +1,81 @@
+import { nowIso, FactScope, FactStatus, isCurrentFact } from '../models.js';
+import { ApplicationError, ApplicationErrorCode } from '../app/application-error.js';
+import { assertStorePort } from '../storage/store-port.js';
+import { writePersonalFact } from './lens-fact-writer.js';
+import { assertSafeLensTexts, requirePersonalSpace } from './lens-input.js';
+
+const ACTIONS = new Set(['replace', 'reject', 'deprecate']);
+const CORRECTABLE_STATUSES = new Set([
+  FactStatus.CONFIRMED,
+  FactStatus.OBSERVED,
+  FactStatus.SUGGESTED
+]);
+const IMMEDIATE = { mode: 'immediate' };
+
+export class LensCorrectionService {
+  constructor(store) {
+    assertStorePort(store);
+    this.store = store;
+  }
+
+  correctUserFact({ personalSpaceId, factId, action, value, sourceText, sourceKind = 'correction' }) {
+    if (!ACTIONS.has(action)) {
+      throw new ApplicationError(ApplicationErrorCode.VALIDATION, `Unknown correction action: ${action}`);
+    }
+
+    requirePersonalSpace(this.store, personalSpaceId);
+    assertSafeLensTexts([sourceText]);
+    if (action === 'replace') assertSafeLensTexts([value]);
+
+    const original = this.#requireCurrentFact(personalSpaceId, factId);
+    if (action === 'replace') assertSafeLensTexts([original.subject, original.predicate]);
+
+    return this.store.transaction(() => {
+      requirePersonalSpace(this.store, personalSpaceId);
+      const current = this.#requireCurrentFact(personalSpaceId, factId);
+      assertSafeLensTexts([sourceText]);
+      if (action === 'replace') assertSafeLensTexts([current.subject, current.predicate, value]);
+
+      const episode = this.store.addEpisode(
+        personalSpaceId,
+        sourceKind,
+        sourceText,
+        null,
+        { kind: 'lens_correction', factId: current.id, action }
+      );
+
+      if (action === 'replace') {
+        const fact = writePersonalFact(this.store, {
+          spaceId: personalSpaceId,
+          subject: current.subject,
+          predicate: current.predicate,
+          object: value,
+          sourceEpisodeId: episode.id,
+          status: FactStatus.CONFIRMED,
+          sensitivity: current.sensitivity,
+          confidence: current.confidence,
+          scope: FactScope.PERSONAL
+        });
+        this.store.invalidateFact(current.id, fact.id);
+        return { episode, fact, replacedFact: this.store.getFact(current.id) };
+      }
+
+      const fact = this.store.updateFact(current.id, {
+        status: action === 'reject' ? FactStatus.REJECTED : FactStatus.DEPRECATED,
+        invalidAt: nowIso()
+      });
+      return { episode, fact };
+    }, IMMEDIATE);
+  }
+
+  #requireCurrentFact(personalSpaceId, factId) {
+    const fact = this.store.getFact(factId);
+    if (!fact || fact.spaceId !== personalSpaceId || fact.scope !== FactScope.PERSONAL) {
+      throw new ApplicationError(ApplicationErrorCode.NOT_FOUND, `Lens fact not found: ${factId}`);
+    }
+    if (!isCurrentFact(fact) || !CORRECTABLE_STATUSES.has(fact.status)) {
+      throw new ApplicationError(ApplicationErrorCode.VALIDATION, `Lens fact is not current: ${factId}`);
+    }
+    return fact;
+  }
+}

package/src/lens/lens-fact-writer.js

@@ -0,0 +1,85 @@
+import { FactScope, FactStatus } from '../models.js';
+
+const STATUS_RANK = Object.freeze({
+  [FactStatus.SUGGESTED]: 1,
+  [FactStatus.OBSERVED]: 2,
+  [FactStatus.CONFIRMED]: 3
+});
+
+export function writePersonalFact(store, input) {
+  const equivalent = equivalentCurrentFacts(store, input);
+  const fact = store.addFact({
+    spaceId: input.spaceId,
+    subject: input.subject,
+    predicate: input.predicate,
+    object: input.object,
+    sourceEpisodeId: input.sourceEpisodeId,
+    status: input.status,
+    confidence: input.confidence,
+    sensitivity: input.sensitivity,
+    scope: FactScope.PERSONAL
+  });
+  const candidates = [...equivalent, fact];
+  const winner = candidates.reduce(strongerOrLater);
+  for (const candidate of candidates) {
+    if (candidate.id !== winner.id) store.invalidateFact(candidate.id, winner.id);
+  }
+  return store.getFact(fact.id);
+}
+
+export function writePersonalFactSpecs(store, input) {
+  const facts = [];
+  for (const spec of input.specs) {
+    if (spec.kind === 'replacement') {
+      facts.push(...writeReplacement(store, input, spec));
+      continue;
+    }
+    facts.push(writePersonalFact(store, {
+      ...input,
+      subject: spec.subject ?? input.subject,
+      predicate: spec.predicate,
+      object: spec.object,
+      status: FactStatus.CONFIRMED
+    }));
+  }
+  return facts;
+}
+
+function writeReplacement(store, input, spec) {
+  const subject = spec.subject ?? input.subject;
+  const oldFacts = store.currentFacts(input.spaceId).filter(
+    (fact) => fact.subject === subject && fact.object === spec.oldValue
+  );
+  const predicates = oldFacts.length
+    ? [...new Set(oldFacts.map((fact) => fact.predicate))]
+    : ['has_replacement'];
+  return predicates.map((predicate) => {
+    const fact = writePersonalFact(store, {
+      ...input,
+      subject,
+      predicate,
+      object: spec.newValue,
+      status: FactStatus.CONFIRMED
+    });
+    for (const previous of oldFacts.filter((item) => item.predicate === predicate)) {
+      store.invalidateFact(previous.id, fact.id);
+    }
+    return fact;
+  });
+}
+
+function equivalentCurrentFacts(store, input) {
+  return store.currentFacts(input.spaceId).filter((fact) =>
+    fact.subject === input.subject &&
+    fact.predicate === input.predicate &&
+    fact.object === input.object
+  );
+}
+
+function strongerOrLater(current, candidate) {
+  return rank(candidate.status) >= rank(current.status) ? candidate : current;
+}
+
+function rank(status) {
+  return STATUS_RANK[status] ?? 0;
+}

package/src/lens/lens-input.js

@@ -0,0 +1,62 @@
+import { ApplicationError, ApplicationErrorCode } from '../app/application-error.js';
+import { Sensitivity, SpaceKind } from '../models.js';
+import { detectSensitiveContent } from '../security/sensitive-content.js';
+
+export function validateLensWriteInput(
+  store,
+  { personalSpaceId, texts, sensitivity, confidence }
+) {
+  requirePersonalSpace(store, personalSpaceId);
+  assertSafeLensTexts(texts);
+  if (!Object.values(Sensitivity).includes(sensitivity)) {
+    throw new ApplicationError(
+      ApplicationErrorCode.VALIDATION,
+      `Invalid sensitivity: ${sensitivity}`
+    );
+  }
+  if (
+    typeof confidence !== 'number' ||
+    !Number.isFinite(confidence) ||
+    confidence < 0 ||
+    confidence > 1
+  ) {
+    throw new ApplicationError(
+      ApplicationErrorCode.VALIDATION,
+      'Confidence must be between 0 and 1'
+    );
+  }
+}
+
+export function requirePersonalSpace(store, spaceId) {
+  const space = store.getSpace(spaceId);
+  if (!space) {
+    throw new ApplicationError(
+      ApplicationErrorCode.NOT_FOUND,
+      `Personal space not found: ${spaceId}`
+    );
+  }
+  if (space.kind !== SpaceKind.PERSONAL) {
+    throw new ApplicationError(
+      ApplicationErrorCode.VALIDATION,
+      `Space is not personal: ${spaceId}`
+    );
+  }
+  return space;
+}
+
+export function assertSafeLensTexts(texts) {
+  for (const text of texts) {
+    if (typeof text !== 'string' || text.length === 0) {
+      throw new ApplicationError(
+        ApplicationErrorCode.VALIDATION,
+        'Lens text must be a nonempty string'
+      );
+    }
+    if (detectSensitiveContent(text).restricted) {
+      throw new ApplicationError(
+        ApplicationErrorCode.VALIDATION,
+        'Sensitive content is not allowed in Personal Lens'
+      );
+    }
+  }
+}

package/src/lens/lens-query-projection.js

@@ -0,0 +1,87 @@
+import { FactScope, FactStatus, isCurrentFact } from '../models.js';
+import { isRestrictedEpisode } from '../security/episode-visibility.js';
+import { isRestrictedFact } from '../security/fact-visibility.js';
+
+export function isInjectableFact(
+  fact,
+  { includeObserved, includeSuggested, includeRestricted }
+) {
+  if (fact.invalidAt) return false;
+  if (fact.status === FactStatus.CONFIRMED) return isVisibleFact(fact, includeRestricted);
+  if (fact.status === FactStatus.OBSERVED && includeObserved) {
+    return isVisibleFact(fact, includeRestricted);
+  }
+  if (fact.status === FactStatus.SUGGESTED && includeSuggested) {
+    return isVisibleFact(fact, includeRestricted);
+  }
+  return false;
+}
+
+export function isSearchableFact(fact, { includeHistorical, includeRestricted }) {
+  return (includeHistorical || isCurrentFact(fact)) && isVisibleFact(fact, includeRestricted);
+}
+
+export function buildCorrectionIndex(groups, personalSpaceId, includeRestricted) {
+  const index = new Map();
+  for (const group of groups) {
+    if (typeof group.factId !== 'string') continue;
+    const episodes = group.episodes.filter((episode) =>
+      episode.spaceId === personalSpaceId &&
+      episode.metadata?.kind === 'lens_correction' &&
+      episode.metadata?.factId === group.factId &&
+      (includeRestricted || !isRestrictedEpisode(episode))
+    );
+    index.set(group.factId, {
+      episodes,
+      truncated: group.truncated === true || episodes.length < group.episodes.length
+    });
+  }
+  return index;
+}
+
+export function projectSearchFact(
+  store,
+  fact,
+  evidenceEpisode,
+  personalSpaceId,
+  correctionIndex,
+  includeRestricted
+) {
+  const sourceEpisode = safePersonalEpisode(
+    evidenceEpisode,
+    personalSpaceId,
+    includeRestricted
+  );
+  const replacement = fact.replacedByFactId ? store.getFact(fact.replacedByFactId) : null;
+  const replacementFact = replacement?.spaceId === personalSpaceId &&
+    replacement.scope === FactScope.PERSONAL &&
+    isVisibleFact(replacement, includeRestricted)
+    ? replacement
+    : null;
+  return {
+    fact: projectSearchFactDto(fact, sourceEpisode, replacementFact),
+    current: isCurrentFact(fact),
+    sourceEpisode,
+    replacementFact,
+    correctionEpisodes: correctionIndex.get(fact.id)?.episodes ?? [],
+    correctionEpisodesTruncated: correctionIndex.get(fact.id)?.truncated ?? false
+  };
+}
+
+function projectSearchFactDto(fact, sourceEpisode, replacementFact) {
+  return {
+    ...fact,
+    sourceEpisodeId: sourceEpisode?.id ?? null,
+    replacedByFactId: replacementFact?.id ?? null
+  };
+}
+
+function isVisibleFact(fact, includeRestricted) {
+  return includeRestricted === true || !isRestrictedFact(fact);
+}
+
+export function safePersonalEpisode(episode, personalSpaceId, includeRestricted) {
+  if (!episode || episode.spaceId !== personalSpaceId) return null;
+  if (!includeRestricted && isRestrictedEpisode(episode)) return null;
+  return episode;
+}

package/src/lens/lens-query-ranking.js

@@ -0,0 +1,62 @@
+import { FactStatus } from '../models.js';
+
+export { compareSearchFacts } from '../storage/fact-search-order.js';
+
+export const BOUNDARY_PREDICATES = Object.freeze([
+  'has_boundary',
+  'has_access_boundary',
+  'does_not_want',
+  'avoids',
+  'forbids'
+]);
+export const BOUNDARY_PREFIXES = Object.freeze([
+  'boundary_',
+  'does_not_',
+  'avoids_',
+  'forbids_',
+  'must_not_'
+]);
+
+const STATUS_RANK = Object.freeze({
+  [FactStatus.SUGGESTED]: 1,
+  [FactStatus.OBSERVED]: 2,
+  [FactStatus.CONFIRMED]: 3
+});
+
+export function taskTokens(task) {
+  return new Set(normalizedTokens(task));
+}
+
+export function compareLensFacts(left, right, terms) {
+  return compareNumber(isBoundary(right), isBoundary(left)) ||
+    compareNumber(termMatches(right, terms), termMatches(left, terms)) ||
+    compareNumber(STATUS_RANK[right.status] ?? 0, STATUS_RANK[left.status] ?? 0) ||
+    compareText(right.validAt, left.validAt) ||
+    compareText(left.id, right.id);
+}
+
+function termMatches(fact, terms) {
+  const factTokens = new Set(normalizedTokens(`${fact.subject} ${fact.predicate} ${fact.object}`));
+  let matches = 0;
+  for (const term of terms) if (factTokens.has(term)) matches += 1;
+  return matches;
+}
+
+function normalizedTokens(text) {
+  return String(text).normalize('NFKC').toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+
+function isBoundary(fact) {
+  return BOUNDARY_PREDICATES.includes(fact.predicate) ||
+    BOUNDARY_PREFIXES.some((prefix) => fact.predicate.startsWith(prefix));
+}
+
+function compareNumber(left, right) {
+  return left - right;
+}
+
+function compareText(left, right) {
+  const first = String(left ?? '');
+  const second = String(right ?? '');
+  return first === second ? 0 : (first < second ? -1 : 1);
+}