@openparser/lineage 1.0.2

package/src/activity.ts

@@ -0,0 +1,65 @@
+import { z } from 'zod';
+import { AttributesSchema, JsonValueSchema, LineageIdSchema, TimestampSchema } from './common';
+
+/** How an agent relates to an activity (PROV wasAssociatedWith). */
+export const AssociationSchema = z
+  .object({
+    agent: LineageIdSchema,
+    role: z.string().trim().min(1).max(128).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Association = z.infer<typeof AssociationSchema>;
+
+export const ACTIVITY_STATUSES = ['scheduled', 'started', 'ended', 'failed', 'cancelled'] as const;
+export const ActivityStatusSchema = z.enum(ACTIVITY_STATUSES);
+export type ActivityStatus = z.infer<typeof ActivityStatusSchema>;
+
+/**
+ * Optional description of the concrete implementation that performed an
+ * activity (library, model, service, script, binary, …).
+ */
+export const ImplementationSchema = z
+  .object({
+    type: z.string().trim().min(1).max(128).optional(),
+    name: z.string().trim().min(1).max(512).optional(),
+    version: z.string().trim().min(1).max(128).optional(),
+    uri: z.string().trim().min(1).max(4096).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Implementation = z.infer<typeof ImplementationSchema>;
+
+/**
+ * A process that used and/or generated entities. `type` is an open string so
+ * profiles can introduce domain-specific activity types without a format bump.
+ *
+ * Map keys in `LineageDocument.activities` are the authoritative activity ids.
+ */
+export const ActivitySchema = z
+  .object({
+    type: z.string().trim().min(1).max(256),
+    name: z.string().trim().min(1).max(512).optional(),
+    status: ActivityStatusSchema.optional(),
+    startedAt: TimestampSchema.optional(),
+    endedAt: TimestampSchema.optional(),
+    implementation: ImplementationSchema.optional(),
+    parameters: z.record(z.string().min(1), JsonValueSchema).optional(),
+    associations: z.array(AssociationSchema).max(64).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Activity = z.infer<typeof ActivitySchema>;
+
+/**
+ * An agent (person, organization, software, service) that acts or is
+ * attributed. Map keys in `LineageDocument.agents` are the authoritative ids.
+ */
+export const AgentSchema = z
+  .object({
+    type: z.string().trim().min(1).max(128).optional(),
+    name: z.string().trim().min(1).max(512).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Agent = z.infer<typeof AgentSchema>;

package/src/approval.ts

@@ -0,0 +1,28 @@
+import { z } from 'zod';
+import { AttributesSchema, LineageIdSchema } from './common';
+
+/**
+ * Who currently stands behind this entity's value.
+ *
+ * An assertion names an agent and a time; it does not change the entity.
+ * OpenParser extraction review projects this from confirmations rather than
+ * storing a separate approve event: an actor counts when their newest live
+ * confirmation on this path names the value this entity holds. Confirming a
+ * matching value adds standing without minting a new entity; confirming a
+ * different value mints a new entity, and standing on the predecessor is left
+ * behind. That is why a sign-off never describes a value nobody confirmed.
+ *
+ * Standing is per entity and does not propagate. Confirming a field says
+ * nothing about the region it was read from — that region carries its own
+ * standing if someone confirms it.
+ */
+export const ApprovalAssertionSchema = z
+  .object({
+    /** Agent in `LineageDocument.agents` that signed off. */
+    agent: LineageIdSchema,
+    at: z.string().datetime(),
+    note: z.string().trim().min(1).max(2_000).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type ApprovalAssertion = z.infer<typeof ApprovalAssertionSchema>;

package/src/builder.ts

@@ -0,0 +1,129 @@
+import type { Activity, Agent } from './activity';
+import { LINEAGE_FORMAT } from './common';
+import type { ConfidenceAssertion } from './confidence';
+import {
+  type Derivation,
+  type DerivationInput,
+  type Transformation,
+  derivationId,
+} from './derivation';
+import type { LineageDocument } from './document';
+import { parseLineageDocument } from './document';
+import type { Entity } from './entity';
+import type { Relation } from './relation';
+
+type EntityInput = Entity;
+type ActivityInput = Activity;
+type AgentInput = Agent;
+
+/**
+ * Ergonomic builder for `lineage@1` documents. Validates on {@link build}.
+ */
+export class LineageBuilder {
+  private id?: string;
+  private profiles?: string[];
+  private readonly entities = new Map<string, EntityInput>();
+  private readonly activities = new Map<string, ActivityInput>();
+  private readonly agents = new Map<string, AgentInput>();
+  private readonly derivations: Derivation[] = [];
+  private relations?: Relation[];
+  private outputs: string[] = [];
+  private attributes?: LineageDocument['attributes'];
+
+  withId(id: string): this {
+    this.id = id;
+    return this;
+  }
+
+  withProfiles(...profiles: string[]): this {
+    this.profiles = [...(this.profiles ?? []), ...profiles];
+    return this;
+  }
+
+  withAttributes(attributes: NonNullable<LineageDocument['attributes']>): this {
+    this.attributes = attributes;
+    return this;
+  }
+
+  entity(id: string, entity: EntityInput): this {
+    if (this.entities.has(id)) {
+      throw new Error(`entity "${id}" already registered`);
+    }
+    this.entities.set(id, entity);
+    return this;
+  }
+
+  activity(id: string, activity: ActivityInput): this {
+    if (this.activities.has(id)) {
+      throw new Error(`activity "${id}" already registered`);
+    }
+    this.activities.set(id, activity);
+    return this;
+  }
+
+  agent(id: string, agent: AgentInput): this {
+    if (this.agents.has(id)) {
+      throw new Error(`agent "${id}" already registered`);
+    }
+    this.agents.set(id, agent);
+    return this;
+  }
+
+  /**
+   * Record that `activityId` produced `outputId` from `inputs`.
+   */
+  derive(args: {
+    /** Optional, as in the wire format: the output already identifies it. */
+    id?: string;
+    output: string;
+    activity: string;
+    inputs?: Array<DerivationInput | string>;
+    transformation?: Transformation;
+    confidence?: ConfidenceAssertion;
+    attributes?: Derivation['attributes'];
+  }): this {
+    const id = derivationId(args);
+    if (this.derivations.some((d) => derivationId(d) === id)) {
+      throw new Error(`derivation "${id}" already registered`);
+    }
+    const inputs: DerivationInput[] = (args.inputs ?? []).map((input) =>
+      typeof input === 'string' ? { entity: input, effect: 'direct' as const } : input
+    );
+    this.derivations.push({
+      ...(args.id ? { id: args.id } : {}),
+      output: args.output,
+      activity: args.activity,
+      inputs,
+      ...(args.transformation ? { transformation: args.transformation } : {}),
+      ...(args.confidence ? { confidence: args.confidence } : {}),
+      ...(args.attributes ? { attributes: args.attributes } : {}),
+    });
+    return this;
+  }
+
+  relation(relation: Relation): this {
+    this.relations = [...(this.relations ?? []), relation];
+    return this;
+  }
+
+  withOutputs(...entityIds: string[]): this {
+    this.outputs = [...this.outputs, ...entityIds];
+    return this;
+  }
+
+  /** Assemble and strictly validate the document. */
+  build(): LineageDocument {
+    return parseLineageDocument({
+      format: LINEAGE_FORMAT,
+      ...(this.id ? { id: this.id } : {}),
+      ...(this.profiles && this.profiles.length > 0 ? { profiles: this.profiles } : {}),
+      entities: Object.fromEntries(this.entities),
+      activities: Object.fromEntries(this.activities),
+      agents: Object.fromEntries(this.agents),
+      derivations: this.derivations,
+      ...(this.relations && this.relations.length > 0 ? { relations: this.relations } : {}),
+      outputs: this.outputs,
+      ...(this.attributes ? { attributes: this.attributes } : {}),
+    });
+  }
+}

package/src/common.ts

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+
+/** Non-empty identifier used as a map key or cross-reference. */
+export const LineageIdSchema = z.string().trim().min(1).max(512);
+export type LineageId = z.infer<typeof LineageIdSchema>;
+
+/** JSON-safe attribute bags and free-form parameter objects. */
+export const JsonValueSchema = z.json();
+export type JsonValue = z.infer<typeof JsonValueSchema>;
+
+export const AttributesSchema = z.record(z.string().min(1), JsonValueSchema);
+export type Attributes = z.infer<typeof AttributesSchema>;
+
+/**
+ * Structural equality for JSON values: whether two values say the same thing.
+ * Object key order is not part of what a JSON value says, so it is normalised
+ * away rather than compared.
+ */
+export function jsonValuesEqual(left: JsonValue, right: JsonValue): boolean {
+  if (left === right) return true;
+  if (left === null || right === null) return false;
+  if (typeof left !== 'object' || typeof right !== 'object') return false;
+  return canonicalJson(left) === canonicalJson(right);
+}
+
+function canonicalJson(value: JsonValue): string {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
+  const entries = Object.entries(value)
+    .filter(([, member]) => member !== undefined)
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+    .map(([key, member]) => `${JSON.stringify(key)}:${canonicalJson(member as JsonValue)}`);
+  return `{${entries.join(',')}}`;
+}
+
+/**
+ * RFC 6901 JSON Pointer. The empty string addresses the whole document.
+ * JSONPath is useful for queries, but pointers give lineage entities one
+ * unambiguous location.
+ */
+export const JsonPointerSchema = z
+  .string()
+  .max(2048)
+  .regex(/^(?:\/(?:[^~/]|~0|~1)*)*$/, 'path must be an RFC 6901 JSON Pointer');
+export type JsonPointer = z.infer<typeof JsonPointerSchema>;
+
+/** Wire format tag for this package. */
+export const LINEAGE_FORMAT = 'lineage@1' as const;
+export const LineageFormatSchema = z.literal(LINEAGE_FORMAT);
+export type LineageFormat = z.infer<typeof LineageFormatSchema>;
+
+/**
+ * RFC 3339 timestamp string. Offset is required (`Z` or `±HH:mm`). Calendar
+ * dates that are not real days are rejected rather than rolled over.
+ * Start/end ordering is enforced at the document graph level.
+ */
+export const TimestampSchema = z.iso.datetime({ offset: true }).max(64);
+export type Timestamp = z.infer<typeof TimestampSchema>;

package/src/confidence.ts

@@ -0,0 +1,72 @@
+import { z } from 'zod';
+import { AttributesSchema, LineageIdSchema } from './common';
+
+/**
+ * Explicit numeric scale for a confidence score.
+ *
+ * Scores are not forced into 0..1 so original provider scales remain
+ * representable without lossy normalization.
+ */
+export const ConfidenceScaleSchema = z
+  .object({
+    min: z.number().finite(),
+    max: z.number().finite(),
+  })
+  .strict()
+  .superRefine((scale, ctx) => {
+    if (!(scale.max > scale.min)) {
+      ctx.addIssue({
+        code: 'custom',
+        message: 'confidence scale max must be greater than min',
+        path: ['max'],
+      });
+    }
+  });
+export type ConfidenceScale = z.infer<typeof ConfidenceScaleSchema>;
+
+export const CONFIDENCE_KINDS = ['reported', 'derived', 'assessed'] as const;
+export const ConfidenceKindSchema = z.enum(CONFIDENCE_KINDS);
+export type ConfidenceKind = z.infer<typeof ConfidenceKindSchema>;
+
+export const ConfidenceSourceSchema = z
+  .object({
+    type: z.enum(['entity', 'activity', 'agent']),
+    id: LineageIdSchema,
+  })
+  .strict();
+export type ConfidenceSource = z.infer<typeof ConfidenceSourceSchema>;
+
+/**
+ * Generic confidence assertion attached to entities, derivations, or other
+ * statements. `score` must lie within `scale`; comparison across uncalibrated
+ * assertions is not implied.
+ */
+export const ConfidenceAssertionSchema = z
+  .object({
+    score: z.number().finite(),
+    scale: ConfidenceScaleSchema,
+    kind: ConfidenceKindSchema,
+    /** What the score measures (e.g. recognition, classification, quality). */
+    scope: z.string().trim().min(1).max(128),
+    /** Optional finer grain within scope (e.g. field, page, token). */
+    granularity: z.string().trim().min(1).max(128).optional(),
+    /** When false, scores must not be treated as calibrated probabilities. */
+    calibrated: z.boolean().default(false),
+    /** Entities, activities, or agents that substantiate or produced this assertion. */
+    sources: z.array(ConfidenceSourceSchema).max(256).optional(),
+    method: z.string().trim().min(1).max(256).optional(),
+    sampleCount: z.number().int().positive().optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict()
+  .superRefine((assertion, ctx) => {
+    const { score, scale } = assertion;
+    if (score < scale.min || score > scale.max) {
+      ctx.addIssue({
+        code: 'custom',
+        message: `confidence score ${score} is outside scale [${scale.min}, ${scale.max}]`,
+        path: ['score'],
+      });
+    }
+  });
+export type ConfidenceAssertion = z.infer<typeof ConfidenceAssertionSchema>;

package/src/derivation.ts

@@ -0,0 +1,81 @@
+import { z } from 'zod';
+import { AttributesSchema, LineageIdSchema } from './common';
+import { ConfidenceAssertionSchema } from './confidence';
+
+export const DERIVATION_EFFECTS = ['direct', 'indirect'] as const;
+export const MAX_DERIVATION_INPUTS = 1_024;
+export const DerivationEffectSchema = z.enum(DERIVATION_EFFECTS);
+export type DerivationEffect = z.infer<typeof DerivationEffectSchema>;
+
+/**
+ * An input entity reference on a derivation, with optional role and whether
+ * the influence on the output is direct or indirect (PROV: Revision /
+ * Quotation / PrimarySource style distinctions are profile-level; here we
+ * keep the general direct/indirect effect flag).
+ *
+ * `confidence` measures this input's contribution to this one output, which is
+ * not the same statement as the input entity's own confidence. A shared source
+ * region is read once but cited by several outputs, and each output may draw on
+ * a different, differently legible part of it — so how well one output was
+ * recognized in that region belongs on the edge, not on the region.
+ */
+export const DerivationInputSchema = z
+  .object({
+    entity: LineageIdSchema,
+    role: z.string().trim().min(1).max(128).optional(),
+    effect: DerivationEffectSchema.default('direct'),
+    confidence: ConfidenceAssertionSchema.optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type DerivationInput = z.infer<typeof DerivationInputSchema>;
+
+/**
+ * Optional descriptor of how inputs were transformed into the output.
+ *
+ * `type` is an open string. OpenLineage's column-lineage vocabulary —
+ * `identity`, `transformation`, `aggregation`, `join`, `group_by`, `filter`,
+ * `sort`, `window`, `conditional` — reads well here, and profiles may use their
+ * own namespaced values instead.
+ */
+export const TransformationSchema = z
+  .object({
+    type: z.string().trim().min(1).max(128).optional(),
+    description: z.string().trim().min(1).max(4096).optional(),
+    expression: z.string().trim().min(1).max(8192).optional(),
+    masking: z.boolean().optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Transformation = z.infer<typeof TransformationSchema>;
+
+/**
+ * Output-centric derivation: one activity produced exactly one output entity
+ * from zero or more typed inputs. A derived entity has at most one producer
+ * derivation in a document (enforced by graph validation).
+ *
+ * `output` already identifies the derivation uniquely, so `id` is optional and
+ * normally omitted — use {@link derivationId} to resolve one. Producers with
+ * their own stable identifiers may still set it.
+ */
+export const DerivationSchema = z
+  .object({
+    id: LineageIdSchema.optional(),
+    output: LineageIdSchema,
+    activity: LineageIdSchema,
+    inputs: z.array(DerivationInputSchema).max(MAX_DERIVATION_INPUTS).default([]),
+    transformation: TransformationSchema.optional(),
+    confidence: ConfidenceAssertionSchema.optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Derivation = z.infer<typeof DerivationSchema>;
+
+/**
+ * Stable identifier for a derivation: its explicit `id` when the producer set
+ * one, otherwise `derivation:<output>`. Because one derivation produces one
+ * output, that fallback is unique within a document.
+ */
+export function derivationId(derivation: Pick<Derivation, 'id' | 'output'>): string {
+  return derivation.id ?? `derivation:${derivation.output}`;
+}

package/src/document.ts

@@ -0,0 +1,82 @@
+import { z } from 'zod';
+import { ActivitySchema, AgentSchema } from './activity';
+import { AttributesSchema, LineageFormatSchema, LineageIdSchema } from './common';
+import { DerivationSchema } from './derivation';
+import { EntitySchema } from './entity';
+import { RelationSchema } from './relation';
+import { LineageValidationError, validateLineageGraph } from './validate';
+
+/**
+ * Structural Zod schema for a `lineage@1` document (shape only).
+ * Use {@link LineageDocumentSchema} or {@link parseLineageDocument} for
+ * shape + strict graph validation.
+ */
+export const LineageDocumentShapeSchema = z
+  .object({
+    format: LineageFormatSchema,
+    id: LineageIdSchema.optional(),
+    /** Profile URIs or short names declaring extension semantics. */
+    profiles: z.array(z.string().trim().min(1).max(1024)).max(64).optional(),
+    entities: z.record(LineageIdSchema, EntitySchema).default({}),
+    activities: z.record(LineageIdSchema, ActivitySchema).default({}),
+    agents: z.record(LineageIdSchema, AgentSchema).default({}),
+    derivations: z.array(DerivationSchema).max(100_000).default([]),
+    relations: z.array(RelationSchema).max(100_000).default([]),
+    /** Entity ids that are primary outputs of this document. */
+    outputs: z.array(LineageIdSchema).max(10_000).default([]),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+
+export type LineageDocument = z.infer<typeof LineageDocumentShapeSchema>;
+
+/**
+ * Parse and strictly validate a lineage document (shape + graph invariants).
+ */
+export function parseLineageDocument(input: unknown): LineageDocument {
+  const doc = LineageDocumentShapeSchema.parse(input);
+  validateLineageGraph(doc);
+  return doc;
+}
+
+/**
+ * Safe parse with Zod issues and/or graph validation errors.
+ */
+export function safeParseLineageDocument(
+  input: unknown
+):
+  | { success: true; data: LineageDocument }
+  | { success: false; error: z.ZodError | LineageValidationError } {
+  const shape = LineageDocumentShapeSchema.safeParse(input);
+  if (!shape.success) {
+    return { success: false, error: shape.error };
+  }
+  try {
+    validateLineageGraph(shape.data);
+    return { success: true, data: shape.data };
+  } catch (error) {
+    if (error instanceof LineageValidationError) {
+      return { success: false, error };
+    }
+    throw error;
+  }
+}
+
+/**
+ * Zod schema that applies graph validation via `superRefine`. Prefer
+ * {@link parseLineageDocument} when you want thrown `LineageValidationError`
+ * with structured issue lists.
+ */
+export const LineageDocumentSchema = LineageDocumentShapeSchema.superRefine((doc, ctx) => {
+  try {
+    validateLineageGraph(doc);
+  } catch (error) {
+    if (error instanceof LineageValidationError) {
+      for (const message of error.issues) {
+        ctx.addIssue({ code: 'custom', message });
+      }
+      return;
+    }
+    throw error;
+  }
+});

package/src/entity.ts

@@ -0,0 +1,87 @@
+import { z } from 'zod';
+import { ApprovalAssertionSchema } from './approval';
+import { AttributesSchema, JsonPointerSchema, JsonValueSchema, LineageIdSchema } from './common';
+import { ConfidenceAssertionSchema } from './confidence';
+
+/** How an agent relates to an entity (PROV wasAttributedTo). */
+export const AttributionSchema = z
+  .object({
+    agent: LineageIdSchema,
+    role: z.string().trim().min(1).max(128).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Attribution = z.infer<typeof AttributionSchema>;
+
+/**
+ * Content-addressing digest for artifacts or values when a cryptographic
+ * fingerprint is available.
+ */
+export const DigestSchema = z
+  .object({
+    algorithm: z.string().trim().min(1).max(64),
+    value: z.string().trim().min(1).max(512),
+  })
+  .strict();
+export type Digest = z.infer<typeof DigestSchema>;
+
+/**
+ * External location of an artifact (URI, media type, optional integrity).
+ */
+export const LocatorSchema = z
+  .object({
+    uri: z.string().trim().min(1).max(4096),
+    mediaType: z.string().trim().min(1).max(256).optional(),
+    digest: DigestSchema.optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Locator = z.infer<typeof LocatorSchema>;
+
+/**
+ * Sub-resource selector within a located artifact (JSON Pointer, XPath,
+ * page/region, byte range, etc.). `type` is open for profiles.
+ */
+export const SelectorSchema = z
+  .object({
+    type: z.string().trim().min(1).max(128),
+    value: JsonValueSchema,
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Selector = z.infer<typeof SelectorSchema>;
+
+export const ENTITY_KINDS = ['value', 'artifact', 'evidence', 'decision', 'collection'] as const;
+export const EntityKindSchema = z.enum(ENTITY_KINDS);
+export type EntityKind = z.infer<typeof EntityKindSchema>;
+
+/**
+ * A node in the derivation DAG: data value, file/blob, supporting evidence,
+ * a decision outcome, or a collection of other entities.
+ *
+ * Map keys in `LineageDocument.entities` are the authoritative entity ids.
+ */
+export const EntitySchema = z
+  .object({
+    kind: EntityKindSchema,
+    name: z.string().trim().min(1).max(512).optional(),
+    /** Inline JSON value when the entity is a materialized datum. */
+    value: JsonValueSchema.optional(),
+    /** Unambiguous location within a larger JSON value (RFC 6901). */
+    path: JsonPointerSchema.optional(),
+    /** Optional JSON Schema (or profile schema object) describing `value`. */
+    schema: z.record(z.string(), JsonValueSchema).optional(),
+    locator: LocatorSchema.optional(),
+    selector: SelectorSchema.optional(),
+    digest: DigestSchema.optional(),
+    confidence: z.array(ConfidenceAssertionSchema).max(64).optional(),
+    /**
+     * Who stands behind this value. Standing is per entity and does not
+     * propagate. OpenParser extraction review projects this from confirmations.
+     */
+    approvals: z.array(ApprovalAssertionSchema).max(64).optional(),
+    attributions: z.array(AttributionSchema).max(64).optional(),
+    attributes: AttributesSchema.optional(),
+  })
+  .strict();
+export type Entity = z.infer<typeof EntitySchema>;