@openparser/lineage 1.0.2

package/src/fields.ts

@@ -0,0 +1,163 @@
+import type { JsonValue } from './common';
+import type { ConfidenceAssertion, ConfidenceKind } from './confidence';
+import type { LineageDocument } from './document';
+import type { Entity } from './entity';
+import { entityAncestors } from './traverse';
+
+/** How a field's confidence came about, or that none was asserted at all. */
+export type ConfidenceOrigin = ConfidenceKind | 'not_reported';
+
+export type LineageField = {
+  id: string;
+  pointer: string;
+  /**
+   * Display-only dotted rendering of `pointer`. Lossy: a property whose name
+   * contains a dot is indistinguishable from nesting. Use `pointer` as the key.
+   */
+  dottedPath: string;
+  value: JsonValue | undefined;
+  confidence: ConfidenceAssertion | undefined;
+  confidenceOrigin: ConfidenceOrigin;
+  entity: Entity;
+};
+
+export type LineageFieldTrace = {
+  field: LineageField;
+  evidence: Array<{ id: string; entity: Entity }>;
+  activities: string[];
+};
+
+/** RFC 6901 pointer for an output entity. `path` is the only field locator. */
+function fieldPointer(entity: Entity): string {
+  return entity.path ?? '';
+}
+
+/** Most relevant confidence assertion: extraction-scoped, else the first. */
+function fieldConfidence(entity: Entity): ConfidenceAssertion | undefined {
+  const assertions = entity.confidence;
+  if (!assertions || assertions.length === 0) return undefined;
+  return assertions.find((assertion) => assertion.scope === 'extraction') ?? assertions[0];
+}
+
+/**
+ * RFC 6901 pointer to dotted display path, unescaping `~1` and `~0`.
+ *
+ * Display only. A property whose name contains a dot is indistinguishable from
+ * nesting, so never use the result as a key.
+ */
+export function dottedPathFromPointer(pointer: string): string {
+  if (pointer === '' || pointer === '/') return '';
+  return pointer
+    .replace(/^\//, '')
+    .split('/')
+    .map((segment) => segment.replace(/~1/g, '/').replace(/~0/g, '~'))
+    .join('.');
+}
+
+/**
+ * Dotted display path back to an RFC 6901 pointer, escaping `~` and `/`.
+ *
+ * Only sound for paths `dottedPathFromPointer` could have produced: a property
+ * name containing a dot round-trips as nesting. Prefer carrying the pointer.
+ */
+export function pointerFromDottedPath(dottedPath: string): string {
+  if (dottedPath === '') return '';
+  return `/${dottedPath
+    .split('.')
+    .map((segment) => segment.replaceAll('~', '~0').replaceAll('/', '~1'))
+    .join('/')}`;
+}
+
+/**
+ * The entity holding a field's current value at `pointer`.
+ *
+ * Precedence is explicit and does not use `Object.entries` insertion order:
+ * 1. An entity at that path listed in `doc.outputs` — after review, `outputs`
+ *    is the current field set (plus a root snapshot / completion decision,
+ *    which do not share field paths). If several outputs match, the earliest
+ *    in `doc.outputs` wins.
+ * 2. Otherwise a `kind: 'decision'` entity at that path (the producer-settled
+ *    value on documents that never went through review). If several decisions
+ *    match, the lexicographically smallest id wins.
+ * 3. Otherwise the lexicographically smallest id among remaining matches.
+ *
+ * `path` is the only field locator, so this is the single supported way to go
+ * from a pointer to an entity id — ids themselves are not a stable contract.
+ */
+export function fieldEntityByPointer(
+  doc: LineageDocument,
+  pointer: string
+): { id: string; entity: Entity } | undefined {
+  const matching = Object.entries(doc.entities).filter(([, entity]) => entity.path === pointer);
+  if (matching.length === 0) return undefined;
+
+  const matchingIds = new Set(matching.map(([id]) => id));
+  const outputId = doc.outputs.find((id) => matchingIds.has(id));
+  if (outputId) {
+    const entity = doc.entities[outputId];
+    if (entity) return { id: outputId, entity };
+  }
+
+  const ranked = matching.toSorted(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0));
+  const decision = ranked.find(([, entity]) => entity.kind === 'decision');
+  const chosen = decision ?? ranked[0];
+  return chosen ? { id: chosen[0], entity: chosen[1] } : undefined;
+}
+
+function toField(id: string, entity: Entity): LineageField {
+  const pointer = fieldPointer(entity);
+  const confidence = fieldConfidence(entity);
+  return {
+    id,
+    pointer,
+    dottedPath: dottedPathFromPointer(pointer),
+    value: entity.value,
+    confidence,
+    confidenceOrigin: confidence ? confidence.kind : 'not_reported',
+    entity,
+  };
+}
+
+/** Output field entities as fields, in `doc.outputs` order.
+ *
+ * Whole-document snapshots (`path === ''`) and other pathless outputs are
+ * omitted — they are the whole result (or a run-level decision), not a field.
+ */
+export function fields(doc: LineageDocument): LineageField[] {
+  const result: LineageField[] = [];
+  for (const id of doc.outputs) {
+    const entity = doc.entities[id];
+    if (!entity) continue;
+    if (fieldPointer(entity) === '') continue;
+    result.push(toField(id, entity));
+  }
+  return result;
+}
+
+/**
+ * Trace a field by JSON Pointer, including upstream evidence and activities.
+ *
+ * Whether that evidence is *sufficient* is a question for whoever produced the
+ * document, since it turns on their own attributes; see the OpenParser profile
+ * for one answer.
+ */
+export function fieldTrace(doc: LineageDocument, pointer: string): LineageFieldTrace | undefined {
+  const field = fields(doc).find((candidate) => candidate.pointer === pointer);
+  if (!field) return undefined;
+
+  const ancestorIds = entityAncestors(doc, field.id);
+  const evidence = ancestorIds
+    .filter((id) => {
+      const entity = doc.entities[id];
+      return entity?.kind === 'collection' || entity?.kind === 'evidence';
+    })
+    .sort()
+    .map((id) => ({ id, entity: doc.entities[id]! }));
+
+  const relevant = new Set([field.id, ...ancestorIds]);
+  const activities = [
+    ...new Set(doc.derivations.filter((d) => relevant.has(d.output)).map((d) => d.activity)),
+  ].sort();
+
+  return { field, evidence, activities };
+}

package/src/index.ts

@@ -0,0 +1,123 @@
+/**
+ * Public `@openparser/lineage` surface for `lineage@1` complete data-derivation DAGs.
+ *
+ * Provider- and domain-agnostic provenance documents grounded in W3C PROV-DM
+ * semantics with ergonomic JSON. Root export only — no subpath exports.
+ */
+
+export {
+  ACTIVITY_STATUSES,
+  ActivitySchema,
+  ActivityStatusSchema,
+  AgentSchema,
+  AssociationSchema,
+  ImplementationSchema,
+  type Activity,
+  type ActivityStatus,
+  type Agent,
+  type Association,
+  type Implementation,
+} from './activity';
+
+export { ApprovalAssertionSchema, type ApprovalAssertion } from './approval';
+
+export { LineageBuilder } from './builder';
+
+export {
+  AttributesSchema,
+  JsonPointerSchema,
+  JsonValueSchema,
+  LINEAGE_FORMAT,
+  LineageFormatSchema,
+  LineageIdSchema,
+  TimestampSchema,
+  jsonValuesEqual,
+  type Attributes,
+  type JsonPointer,
+  type JsonValue,
+  type LineageFormat,
+  type LineageId,
+  type Timestamp,
+} from './common';
+
+export {
+  CONFIDENCE_KINDS,
+  ConfidenceAssertionSchema,
+  ConfidenceKindSchema,
+  ConfidenceScaleSchema,
+  ConfidenceSourceSchema,
+  type ConfidenceAssertion,
+  type ConfidenceKind,
+  type ConfidenceScale,
+  type ConfidenceSource,
+} from './confidence';
+
+export {
+  DERIVATION_EFFECTS,
+  DerivationEffectSchema,
+  DerivationInputSchema,
+  DerivationSchema,
+  MAX_DERIVATION_INPUTS,
+  TransformationSchema,
+  derivationId,
+  type Derivation,
+  type DerivationEffect,
+  type DerivationInput,
+  type Transformation,
+} from './derivation';
+
+export {
+  LineageDocumentSchema,
+  LineageDocumentShapeSchema,
+  parseLineageDocument,
+  safeParseLineageDocument,
+  type LineageDocument,
+} from './document';
+
+export {
+  AttributionSchema,
+  DigestSchema,
+  ENTITY_KINDS,
+  EntityKindSchema,
+  EntitySchema,
+  LocatorSchema,
+  SelectorSchema,
+  type Attribution,
+  type Digest,
+  type Entity,
+  type EntityKind,
+  type Locator,
+  type Selector,
+} from './entity';
+
+export {
+  LINEAGE_PROV_NAMESPACE,
+  lineageIdToProvQualifiedName,
+  toProvJson,
+  type ProvJsonDocument,
+} from './prov';
+
+export {
+  RELATION_TYPES,
+  RelationSchema,
+  RelationTypeSchema,
+  type Relation,
+  type RelationType,
+} from './relation';
+
+export {
+  dottedPathFromPointer,
+  fieldEntityByPointer,
+  fieldTrace,
+  fields,
+  pointerFromDottedPath,
+  type ConfidenceOrigin,
+  type LineageField,
+  type LineageFieldTrace,
+} from './fields';
+
+export { activityLabel } from './labels';
+
+export { entityAncestors, entityDescendants, topologicalEntityOrder } from './traverse';
+
+export { LineageValidationError, validateLineageGraph } from './validate';

package/src/labels.ts

@@ -0,0 +1,13 @@
+import type { Activity } from './activity';
+
+/**
+ * Display label for an activity: an explicit `name` when a producer set one,
+ * otherwise the raw `type`.
+ *
+ * `type` is an open string. Core `lineage@1` does not map vendor activity types
+ * to presentation copy — OpenParser-friendly names live in
+ * `@openparser/lineage/openparser`.
+ */
+export function activityLabel(activity: Activity): string {
+  return activity.name ?? activity.type;
+}

package/src/profile/grounding.ts

@@ -0,0 +1,74 @@
+/**
+ * Whether a field's evidence is sufficient, as the OpenParser profile defines it.
+ *
+ * `lineage@1` records what a value was derived from; it takes no position on when
+ * that is enough, because "enough" depends on what the producer was able to
+ * resolve. OpenParser answers it with one attribute on the producer derivation —
+ * the source ids the model named that could not be resolved to a region — so the
+ * rule reads a `openparser:` key and belongs here rather than in the protocol.
+ */
+
+import type { Derivation } from '../derivation';
+import type { LineageDocument } from '../document';
+import { fieldTrace } from '../fields';
+import { entityAncestors } from '../traverse';
+
+const DROPPED_SOURCES_ATTR = 'openparser:droppedSourceIds';
+
+/** Whether a field is backed by source evidence, partially backed, or unbacked. */
+export type GroundingStatus = 'grounded' | 'partial' | 'ungrounded';
+
+export type FieldGrounding = {
+  status: GroundingStatus;
+  /** Source ids the producer cited but could not resolve to a region. */
+  droppedSourceIds: string[];
+};
+
+/** Source ids the producer named but could not resolve to a region. */
+export function droppedSourceIds(producer: Derivation | undefined): string[] {
+  const dropped = producer?.attributes?.[DROPPED_SOURCES_ATTR];
+  return Array.isArray(dropped) ? dropped.filter((id): id is string => typeof id === 'string') : [];
+}
+
+/**
+ * Grounding is read off the graph rather than stored: the evidence a value
+ * actually reaches, less the sources its producer could not resolve.
+ *
+ * Callers that already walked to the evidence pass the count, since how far a
+ * given consumer follows the graph is its own business.
+ */
+export function groundingStatus(options: {
+  evidenceCount: number;
+  droppedSourceIds: string[];
+}): GroundingStatus {
+  if (options.evidenceCount === 0) return 'ungrounded';
+  return options.droppedSourceIds.length > 0 ? 'partial' : 'grounded';
+}
+
+function collectDroppedSourceIds(doc: LineageDocument, entityIds: Iterable<string>): string[] {
+  const relevant = new Set(entityIds);
+  const seen = new Set<string>();
+  const dropped: string[] = [];
+  for (const derivation of doc.derivations) {
+    if (!relevant.has(derivation.output)) continue;
+    for (const id of droppedSourceIds(derivation)) {
+      if (seen.has(id)) continue;
+      seen.add(id);
+      dropped.push(id);
+    }
+  }
+  return dropped;
+}
+
+/** Grounding for the field at a JSON Pointer, walking the graph to its evidence. */
+export function fieldGrounding(doc: LineageDocument, pointer: string): FieldGrounding | undefined {
+  const trace = fieldTrace(doc, pointer);
+  if (!trace) return undefined;
+
+  const ancestorIds = entityAncestors(doc, trace.field.id);
+  const dropped = collectDroppedSourceIds(doc, [trace.field.id, ...ancestorIds]);
+  return {
+    status: groundingStatus({ evidenceCount: trace.evidence.length, droppedSourceIds: dropped }),
+    droppedSourceIds: dropped,
+  };
+}

package/src/profile/index.ts

@@ -0,0 +1,67 @@
+/**
+ * The OpenParser profile of `lineage@1`, published as `@openparser/lineage/openparser`.
+ *
+ * `lineage@1` is a vendor-neutral record of where values came from. Anything that
+ * depends on how OpenParser in particular scores an OCR word, or on an
+ * `openparser:` attribute, lives here — so the protocol surface stays readable as
+ * a protocol, and a producer that is not OpenParser is never asked to adopt our
+ * conventions to use it.
+ */
+
+export { activityLabel } from './labels';
+
+export {
+  droppedSourceIds,
+  fieldGrounding,
+  groundingStatus,
+  type FieldGrounding,
+  type GroundingStatus,
+} from './grounding';
+
+export {
+  CITED_RECOGNITION_METHOD,
+  CONTEXT_BLEND_WEIGHT,
+  REGION_RECOGNITION_METHOD,
+  aggregateCitedWordConfidences,
+  annotateWordsWithCitation,
+  citedSpanInText,
+  downwardBlendCitedMinimum,
+  type CitedWordAggregate,
+  type ScoredWord,
+} from './recognition';
+
+export {
+  DATE_SOURCE_CATALOG_ID,
+  DATE_SOURCE_CATALOG_V1,
+  DATE_SOURCE_CATALOG_VERSION,
+  FORMAT_ONLY_DATE_TARGETS,
+  MAX_TRANSFORM_INTENT_PARAMETER_BYTES,
+  MAX_TRANSFORM_PROOF_BYTES,
+  TRANSFORM_DOCS_ANCHORS,
+  TRANSFORM_OPERATIONS,
+  TRANSFORM_PROOF_ATTRIBUTE,
+  TRANSFORM_PROOF_OPERAND_KEYS,
+  TRANSFORM_VALIDATORS,
+  TRANSFORM_VERIFICATION_DOCS,
+  TransformClaimSchema,
+  TransformIntentSchema,
+  TransformProofClassSchema,
+  TransformProofSchema,
+  dateTimeFormatIssue,
+  formatOnlyDateTimeParameters,
+  isTrustedTransformDocsHref,
+  parseTransformProof,
+  transformValidatorDescriptor,
+  trustedDateSourceFormats,
+  trustedIntentParametersFitProof,
+  trustedTransformDocsHref,
+  verifyTransform,
+  type DateTimeKind,
+  type TransformClaim,
+  type TransformIntent,
+  type TransformOperation,
+  type TransformProof,
+  type TransformProofClass,
+  type TransformValidatorDescriptor,
+  type TransformVerificationResult,
+} from './transform-validation';

package/src/profile/labels.ts

@@ -0,0 +1,28 @@
+import type { Activity } from '../activity';
+
+/**
+ * Human labels for the activity types this profile emits.
+ *
+ * Documents do not carry generated display strings, so this is where a label is
+ * recovered from `activity.type`. Keeping it here rather than in each dashboard
+ * means one mapping to translate or correct, and it stays available to anyone
+ * reading a document through this library.
+ */
+const ACTIVITY_LABELS: Record<string, string> = {
+  ocr: 'Parse',
+  extract: 'Schema-constrained extraction',
+  transform: 'Transform',
+  grounding_resolution: 'Grounding resolution and validation',
+  'review.confirm': 'Human review',
+};
+
+/**
+ * Display label for an OpenParser activity: an explicit `name` when a producer
+ * set one, otherwise a label for known types, otherwise the raw type.
+ *
+ * `type` is an open string, so unknown types render as themselves rather than
+ * being hidden behind a placeholder.
+ */
+export function activityLabel(activity: Activity): string {
+  return activity.name ?? ACTIVITY_LABELS[activity.type] ?? activity.type;
+}

package/src/profile/recognition.ts

@@ -0,0 +1,149 @@
+/**
+ * How a value's recognition score is derived from the OCR words behind it.
+ *
+ * This is a profile rule, not part of `lineage@1`: the protocol says a score has
+ * a scale, a scope, and a named method, and stays silent on what any particular
+ * method computes. It ships beside the protocol because a producer and a consumer
+ * that disagree here disagree about what a number means. A region cited by two
+ * fields is read once but quoted differently by each, so scoring the region and
+ * scoring a value are different questions — and answering the second with the
+ * first is how a crisply scanned number ends up wearing its neighbor's bad
+ * handwriting.
+ */
+
+/** An OCR token with its recognition score, and whether a value was read from it. */
+export type ScoredWord = {
+  id?: string;
+  text: string;
+  confidence: number;
+  cited?: boolean;
+};
+
+export const CITED_RECOGNITION_METHOD = 'cited_minimum_context_downblend';
+
+/** Scoring a whole region, because no word within it could be narrowed to. */
+export const REGION_RECOGNITION_METHOD = 'region_word_minimum';
+
+/** How far the surrounding text may pull a cited score down. */
+export const CONTEXT_BLEND_WEIGHT = 0.15;
+
+/**
+ * Blend a cited-word minimum with the surrounding context: weaker context may
+ * only pull the score down, and never below what the blend allows. A noisy
+ * neighborhood is evidence that a value might be misread, not a verdict that it
+ * was.
+ */
+export function downwardBlendCitedMinimum(
+  citedMin: number,
+  contextMean: number | undefined
+): number {
+  if (contextMean === undefined) return citedMin;
+  const blended = citedMin * (1 - CONTEXT_BLEND_WEIGHT) + contextMean * CONTEXT_BLEND_WEIGHT;
+  return Math.min(citedMin, blended);
+}
+
+export type CitedWordAggregate = {
+  score: number;
+  minimum: number;
+  maximum: number;
+  sampleCount: number;
+  method: typeof CITED_RECOGNITION_METHOD | typeof REGION_RECOGNITION_METHOD;
+  contextMean?: number;
+  contextSampleCount?: number;
+};
+
+/**
+ * Score one value from the words it was read from, discounted by the words
+ * around it. With nothing marked cited there is nothing to narrow to, so every
+ * word counts, the result is the region's own weakest token, and `method` says
+ * so — a score over the whole region must not claim to have narrowed to a value.
+ */
+export function aggregateCitedWordConfidences(
+  words: readonly ScoredWord[]
+): CitedWordAggregate | undefined {
+  if (words.length === 0) return undefined;
+  const cited = words.filter((word) => word.cited !== false);
+  const context = words.filter((word) => word.cited === false);
+  const active = cited.length > 0 ? cited : words;
+  const activeScores = active.map((word) => word.confidence);
+  const contextMean =
+    cited.length > 0 && context.length > 0
+      ? context.reduce((sum, word) => sum + word.confidence, 0) / context.length
+      : undefined;
+  const citedMin = Math.min(...activeScores);
+
+  return {
+    score: downwardBlendCitedMinimum(citedMin, contextMean),
+    minimum: citedMin,
+    maximum: Math.max(...activeScores),
+    sampleCount: active.length,
+    method: cited.length > 0 ? CITED_RECOGNITION_METHOD : REGION_RECOGNITION_METHOD,
+    ...(contextMean !== undefined ? { contextMean, contextSampleCount: context.length } : {}),
+  };
+}
+
+/** First candidate found in the text wins; pass candidates in priority order. */
+export function citedSpanInText(
+  regionText: string,
+  candidates: ReadonlyArray<string | undefined>
+): { start: number; end: number } {
+  if (!regionText) return { start: -1, end: -1 };
+  for (const candidate of candidates) {
+    const quote = candidate?.trim();
+    if (!quote) continue;
+    let start = regionText.indexOf(quote);
+    if (start < 0) start = regionText.toLowerCase().indexOf(quote.toLowerCase());
+    if (start >= 0) return { start, end: start + quote.length };
+  }
+  return { start: -1, end: -1 };
+}
+
+function wordOverlapsSpan(
+  regionText: string,
+  word: ScoredWord,
+  cursor: number,
+  span: { start: number; end: number }
+): { overlaps: boolean; nextCursor: number } {
+  if (!word.text) return { overlaps: false, nextCursor: cursor };
+  const at = regionText.indexOf(word.text, cursor);
+  if (at < 0) return { overlaps: false, nextCursor: cursor };
+  const wordEnd = at + word.text.length;
+  return {
+    overlaps: span.start >= 0 && at < span.end && wordEnd > span.start,
+    nextCursor: wordEnd,
+  };
+}
+
+/**
+ * Mark the words a value was read from within a region.
+ *
+ * `extractedText` is what the extraction produced, which is always text present
+ * in the document: for a value returned as-is it is the value itself, and for one
+ * the model went on to rewrite — `2024-05-30` from `MAY 30, 2024` — it is the text
+ * that was actually read. Searching for the value instead would find nothing in
+ * the second case. `lineageWordTexts`/`lineageWordIds` name words a lineage
+ * document already attributes to the value, used when no span matches.
+ */
+export function annotateWordsWithCitation<T extends ScoredWord>(
+  regionText: string,
+  words: readonly T[],
+  options?: {
+    extractedText?: string;
+    lineageWordTexts?: ReadonlySet<string>;
+    lineageWordIds?: ReadonlySet<string>;
+  }
+): Array<T & { cited: boolean }> {
+  const span = citedSpanInText(regionText, [options?.extractedText]);
+  const hasQuoteSpan = span.start >= 0;
+  let cursor = 0;
+
+  return words.map((word) => {
+    const { overlaps, nextCursor } = wordOverlapsSpan(regionText, word, cursor, span);
+    cursor = nextCursor;
+    const inLineage = Boolean(
+      (word.id && options?.lineageWordIds?.has(word.id)) ||
+      options?.lineageWordTexts?.has(word.text)
+    );
+    return { ...word, cited: hasQuoteSpan ? overlaps : inLineage };
+  });
+}