@openparser/lineage 1.0.2

package/src/prov.ts

@@ -0,0 +1,205 @@
+import { derivationId } from './derivation';
+import type { LineageDocument } from './document';
+
+/**
+ * Subset of W3C PROV-JSON used for interoperability export.
+ *
+ * Note: PROV-JSON itself is a W3C Member Submission
+ * (https://www.w3.org/Submission/2013/SUBM-prov-json-20130424/).
+ * The conceptual model is PROV-DM, a W3C Recommendation
+ * (https://www.w3.org/TR/prov-dm/).
+ */
+export type ProvJsonDocument = {
+  prefix?: Record<string, string>;
+  entity?: Record<string, Record<string, unknown>>;
+  activity?: Record<string, Record<string, unknown>>;
+  agent?: Record<string, Record<string, unknown>>;
+  wasDerivedFrom?: Record<string, Record<string, unknown>>;
+  wasGeneratedBy?: Record<string, Record<string, unknown>>;
+  used?: Record<string, Record<string, unknown>>;
+  wasAssociatedWith?: Record<string, Record<string, unknown>>;
+  wasAttributedTo?: Record<string, Record<string, unknown>>;
+  hadMember?: Record<string, Record<string, unknown>>;
+  specializationOf?: Record<string, Record<string, unknown>>;
+  alternateOf?: Record<string, Record<string, unknown>>;
+};
+
+/**
+ * Where the `lineage@1` vocabulary is published, so PROV consumers can resolve
+ * the `lineage:` prefix. It names the format, not a producer — a document emitted
+ * by anyone binds the same prefix. Moving it breaks every consumer that has
+ * already resolved a term against it, so it is versioned with the format itself.
+ */
+export const LINEAGE_PROV_NAMESPACE = 'https://openparser.dev/ns/lineage#';
+
+export function lineageIdToProvQualifiedName(id: string): string {
+  const encoded = [...new TextEncoder().encode(id)]
+    .map((byte) => byte.toString(16).padStart(2, '0'))
+    .join('');
+  return `lineage:id_${encoded}`;
+}
+const qname = lineageIdToProvQualifiedName;
+
+/**
+ * Map a validated `lineage@1` document to W3C PROV-JSON concepts.
+ *
+ * Derivations expand to `wasDerivedFrom` (per input), `wasGeneratedBy`
+ * (output ← activity), and `used` (activity ← input). Associations,
+ * attributions, and membership follow PROV relations.
+ */
+export function toProvJson(doc: LineageDocument): ProvJsonDocument {
+  const out: ProvJsonDocument = {
+    prefix: {
+      prov: 'http://www.w3.org/ns/prov#',
+      lineage: LINEAGE_PROV_NAMESPACE,
+    },
+  };
+
+  const entities: Record<string, Record<string, unknown>> = {};
+  for (const [id, entity] of Object.entries(doc.entities)) {
+    const record: Record<string, unknown> = flattenAttrs(entity.attributes);
+    record['prov:type'] = entity.kind;
+    record['lineage:id'] = id;
+    if (entity.name) record['prov:label'] = entity.name;
+    if (entity.value !== undefined) record['lineage:value'] = entity.value;
+    // The empty pointer addresses the whole document, so absence is the test.
+    if (entity.path !== undefined) record['lineage:path'] = entity.path;
+    if (entity.locator?.uri) record['prov:location'] = entity.locator.uri;
+    if (entity.digest) {
+      record['lineage:digestAlgorithm'] = entity.digest.algorithm;
+      record['lineage:digestValue'] = entity.digest.value;
+    }
+    entities[qname(id)] = record;
+  }
+  if (Object.keys(entities).length > 0) out.entity = entities;
+
+  const activities: Record<string, Record<string, unknown>> = {};
+  for (const [id, activity] of Object.entries(doc.activities)) {
+    const record: Record<string, unknown> = flattenAttrs(activity.attributes);
+    record['prov:type'] = activity.type;
+    record['lineage:id'] = id;
+    if (activity.name) record['prov:label'] = activity.name;
+    if (activity.startedAt) record['prov:startTime'] = activity.startedAt;
+    if (activity.endedAt) record['prov:endTime'] = activity.endedAt;
+    activities[qname(id)] = record;
+  }
+  if (Object.keys(activities).length > 0) out.activity = activities;
+
+  const agents: Record<string, Record<string, unknown>> = {};
+  for (const [id, agent] of Object.entries(doc.agents)) {
+    const record: Record<string, unknown> = flattenAttrs(agent.attributes);
+    record['lineage:id'] = id;
+    if (agent.type) record['prov:type'] = agent.type;
+    if (agent.name) record['prov:label'] = agent.name;
+    agents[qname(id)] = record;
+  }
+  if (Object.keys(agents).length > 0) out.agent = agents;
+
+  const wasDerivedFrom: Record<string, Record<string, unknown>> = {};
+  const wasGeneratedBy: Record<string, Record<string, unknown>> = {};
+  const used: Record<string, Record<string, unknown>> = {};
+  const wasAssociatedWith: Record<string, Record<string, unknown>> = {};
+  const wasAttributedTo: Record<string, Record<string, unknown>> = {};
+  const hadMember: Record<string, Record<string, unknown>> = {};
+  const specializationOf: Record<string, Record<string, unknown>> = {};
+  const alternateOf: Record<string, Record<string, unknown>> = {};
+
+  for (const derivation of doc.derivations) {
+    const id = derivationId(derivation);
+    const genId = qname(`gen:${id}`);
+    wasGeneratedBy[genId] = {
+      'prov:entity': qname(derivation.output),
+      'prov:activity': qname(derivation.activity),
+    };
+
+    for (let i = 0; i < derivation.inputs.length; i++) {
+      const input = derivation.inputs[i]!;
+      const derId = qname(`wdf:${id}:${i}`);
+      wasDerivedFrom[derId] = {
+        'prov:generatedEntity': qname(derivation.output),
+        'prov:usedEntity': qname(input.entity),
+        'prov:activity': qname(derivation.activity),
+        ...(input.role ? { 'prov:role': input.role } : {}),
+        'lineage:effect': input.effect,
+      };
+
+      const usedId = qname(`used:${id}:${i}`);
+      used[usedId] = {
+        'prov:activity': qname(derivation.activity),
+        'prov:entity': qname(input.entity),
+        ...(input.role ? { 'prov:role': input.role } : {}),
+      };
+    }
+  }
+
+  for (const [activityId, activity] of Object.entries(doc.activities)) {
+    for (let i = 0; i < (activity.associations?.length ?? 0); i++) {
+      const association = activity.associations![i]!;
+      wasAssociatedWith[qname(`assoc:${activityId}:${i}`)] = {
+        'prov:activity': qname(activityId),
+        'prov:agent': qname(association.agent),
+        ...(association.role ? { 'prov:role': association.role } : {}),
+      };
+    }
+  }
+
+  for (const [entityId, entity] of Object.entries(doc.entities)) {
+    for (let i = 0; i < (entity.attributions?.length ?? 0); i++) {
+      const attribution = entity.attributions![i]!;
+      wasAttributedTo[qname(`attr:${entityId}:${i}`)] = {
+        'prov:entity': qname(entityId),
+        'prov:agent': qname(attribution.agent),
+        ...(attribution.role ? { 'prov:role': attribution.role } : {}),
+      };
+    }
+  }
+
+  for (let i = 0; i < (doc.relations?.length ?? 0); i++) {
+    const relation = doc.relations![i]!;
+    if (relation.type === 'member_of') {
+      // PROV hadMember: collection → member (inverse of our member_of direction)
+      hadMember[qname(`mem:${i}`)] = {
+        'prov:collection': qname(relation.target),
+        'prov:entity': qname(relation.source),
+      };
+    } else if (relation.type === 'specialization_of') {
+      specializationOf[qname(`spec:${i}`)] = {
+        'prov:specificEntity': qname(relation.source),
+        'prov:generalEntity': qname(relation.target),
+      };
+    } else if (relation.type === 'alternate_of') {
+      alternateOf[qname(`alt:${i}`)] = {
+        'prov:alternate1': qname(relation.source),
+        'prov:alternate2': qname(relation.target),
+      };
+    }
+  }
+
+  if (Object.keys(wasDerivedFrom).length > 0) out.wasDerivedFrom = wasDerivedFrom;
+  if (Object.keys(wasGeneratedBy).length > 0) out.wasGeneratedBy = wasGeneratedBy;
+  if (Object.keys(used).length > 0) out.used = used;
+  if (Object.keys(wasAssociatedWith).length > 0) out.wasAssociatedWith = wasAssociatedWith;
+  if (Object.keys(wasAttributedTo).length > 0) out.wasAttributedTo = wasAttributedTo;
+  if (Object.keys(hadMember).length > 0) out.hadMember = hadMember;
+  if (Object.keys(specializationOf).length > 0) out.specializationOf = specializationOf;
+  if (Object.keys(alternateOf).length > 0) out.alternateOf = alternateOf;
+
+  return out;
+}
+
+/**
+ * Attributes render alongside canonical PROV keys, so callers write them first
+ * and let the canonical keys land last: an attribute named `prov:type` then
+ * loses to the real kind instead of falsifying it.
+ *
+ * Prefixes this document does not declare are folded into `lineage:attribute:*`
+ * rather than emitted as undeclared QNames, which loses the profile namespace.
+ */
+function flattenAttrs(attrs: Record<string, unknown> | undefined): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(attrs ?? {})) {
+    out[key.startsWith('prov:') || key.startsWith('lineage:') ? key : qname(`attribute:${key}`)] =
+      value;
+  }
+  return out;
+}

package/src/relation.ts

@@ -0,0 +1,32 @@
+import { z } from 'zod';
+import { AttributesSchema, LineageIdSchema } from './common';
+
+export const RELATION_TYPES = ['member_of', 'specialization_of', 'alternate_of'] as const;
+export const RelationTypeSchema = z.enum(RELATION_TYPES);
+export type RelationType = z.infer<typeof RelationTypeSchema>;
+
+/**
+ * Semantic entity–entity relations outside the derivation DAG.
+ *
+ * - `member_of`: `source` is a member; `target` must be a `collection` entity
+ * - `specialization_of`: `source` specializes `target` (PROV specializationOf)
+ * - `alternate_of`: `source` and `target` are alternate of each other
+ */
+export const RelationSchema = z
+  .object({
+    type: RelationTypeSchema,
+    source: LineageIdSchema,
+    target: LineageIdSchema,
+    attributes: AttributesSchema.optional(),
+  })
+  .strict()
+  .superRefine((relation, ctx) => {
+    if (relation.source === relation.target) {
+      ctx.addIssue({
+        code: 'custom',
+        message: 'relation source and target must be distinct',
+        path: ['target'],
+      });
+    }
+  });
+export type Relation = z.infer<typeof RelationSchema>;

package/src/traverse.ts

@@ -0,0 +1,91 @@
+import type { LineageDocument } from './document';
+
+/** Build adjacency: entity → entities derived from it (children / descendants direction). */
+function buildForwardEdges(doc: LineageDocument): Map<string, string[]> {
+  const edges = new Map<string, string[]>();
+  for (const id of Object.keys(doc.entities)) edges.set(id, []);
+  for (const derivation of doc.derivations) {
+    if (!(derivation.output in doc.entities)) continue;
+    for (const input of derivation.inputs) {
+      if (!(input.entity in doc.entities)) continue;
+      edges.get(input.entity)!.push(derivation.output);
+    }
+  }
+  return edges;
+}
+
+/** Build adjacency: entity → entities it was derived from (parents / ancestors direction). */
+function buildReverseEdges(doc: LineageDocument): Map<string, string[]> {
+  const edges = new Map<string, string[]>();
+  for (const id of Object.keys(doc.entities)) edges.set(id, []);
+  for (const derivation of doc.derivations) {
+    if (!(derivation.output in doc.entities)) continue;
+    for (const input of derivation.inputs) {
+      if (!(input.entity in doc.entities)) continue;
+      edges.get(derivation.output)!.push(input.entity);
+    }
+  }
+  return edges;
+}
+
+/**
+ * Topological order of entity ids following derivation dependencies
+ * (inputs before outputs). Stable for disconnected nodes by insertion order
+ * of `Object.keys(entities)`.
+ */
+export function topologicalEntityOrder(doc: LineageDocument): string[] {
+  const forward = buildForwardEdges(doc);
+  const indegree = new Map<string, number>();
+  for (const id of Object.keys(doc.entities)) indegree.set(id, 0);
+  for (const [, targets] of forward) {
+    for (const t of targets) {
+      indegree.set(t, (indegree.get(t) ?? 0) + 1);
+    }
+  }
+
+  const queue: string[] = [];
+  for (const id of Object.keys(doc.entities)) {
+    if ((indegree.get(id) ?? 0) === 0) queue.push(id);
+  }
+
+  const order: string[] = [];
+  while (queue.length > 0) {
+    const id = queue.shift()!;
+    order.push(id);
+    for (const next of forward.get(id) ?? []) {
+      const nextDeg = (indegree.get(next) ?? 0) - 1;
+      indegree.set(next, nextDeg);
+      if (nextDeg === 0) queue.push(next);
+    }
+  }
+
+  if (order.length !== Object.keys(doc.entities).length) {
+    throw new Error('entity dependency graph contains a cycle');
+  }
+  return order;
+}
+
+function collectReachable(start: string, edges: Map<string, string[]>): string[] {
+  if (!edges.has(start)) return [];
+  const seen = new Set<string>();
+  const stack = [...(edges.get(start) ?? [])];
+  while (stack.length > 0) {
+    const id = stack.pop()!;
+    if (seen.has(id)) continue;
+    seen.add(id);
+    for (const next of edges.get(id) ?? []) {
+      if (!seen.has(next)) stack.push(next);
+    }
+  }
+  return [...seen];
+}
+
+/** All entities reachable upstream via derivation inputs (transitive). */
+export function entityAncestors(doc: LineageDocument, entityId: string): string[] {
+  return collectReachable(entityId, buildReverseEdges(doc));
+}
+
+/** All entities reachable downstream via derivation outputs (transitive). */
+export function entityDescendants(doc: LineageDocument, entityId: string): string[] {
+  return collectReachable(entityId, buildForwardEdges(doc));
+}

package/src/validate.ts

@@ -0,0 +1,221 @@
+import { derivationId } from './derivation';
+import type { LineageDocument } from './document';
+
+export class LineageValidationError extends Error {
+  readonly issues: string[];
+
+  constructor(issues: string[]) {
+    super(issues.join('; '));
+    this.name = 'LineageValidationError';
+    this.issues = issues;
+  }
+}
+
+/**
+ * Strict graph validation beyond Zod shape checks.
+ *
+ * Ensures globally distinct ids, resolvable references, unique derivation
+ * outputs (one producer), no self-inputs, acyclic entity dependencies,
+ * valid collection membership, and coherent activity timestamps.
+ */
+export function validateLineageGraph(doc: LineageDocument): void {
+  const issues: string[] = [];
+
+  const entityIds = new Set(Object.keys(doc.entities));
+  const activityIds = new Set(Object.keys(doc.activities));
+  const agentIds = new Set(Object.keys(doc.agents));
+
+  const idsOfKind = { entity: entityIds, activity: activityIds, agent: agentIds } as const;
+
+  /** Every cross-reference in a document resolves through here, so none is missed. */
+  const requireRef = (kind: keyof typeof idsOfKind, id: string, where: string) => {
+    if (!idsOfKind[kind].has(id)) issues.push(`${where}: ${kind} "${id}" does not exist`);
+  };
+  const requireConfidenceRefs = (
+    confidence:
+      | { sources?: ReadonlyArray<{ type: keyof typeof idsOfKind; id: string }> }
+      | undefined,
+    where: string
+  ) => {
+    for (const source of confidence?.sources ?? []) {
+      requireRef(source.type, source.id, `${where}.sources`);
+    }
+  };
+
+  // Globally distinct IDs across entity / activity / agent maps.
+  const claimed = new Map<string, string>();
+  for (const [bucket, ids] of [
+    ['entities', entityIds],
+    ['activities', activityIds],
+    ['agents', agentIds],
+  ] as const) {
+    for (const id of ids) {
+      const prior = claimed.get(id);
+      if (prior) {
+        issues.push(`id "${id}" collides across ${prior} and ${bucket}`);
+      } else {
+        claimed.set(id, bucket);
+      }
+    }
+  }
+
+  // Derivation ids must be unique and not collide with map keys.
+  const derivationIds = new Set<string>();
+  const outputProducers = new Map<string, string>();
+
+  for (let i = 0; i < doc.derivations.length; i++) {
+    const derivation = doc.derivations[i]!;
+    const path = `derivations[${i}]`;
+
+    const id = derivationId(derivation);
+
+    if (derivationIds.has(id)) {
+      issues.push(`${path}: duplicate derivation id "${id}"`);
+    }
+    derivationIds.add(id);
+
+    if (claimed.has(id)) {
+      issues.push(`${path}: derivation id "${id}" collides with ${claimed.get(id)}`);
+    }
+
+    if (!entityIds.has(derivation.output)) {
+      issues.push(`${path}: output entity "${derivation.output}" does not exist`);
+    } else {
+      const prior = outputProducers.get(derivation.output);
+      if (prior) {
+        issues.push(
+          `${path}: entity "${derivation.output}" already has producer derivation "${prior}" (one producer per derived output)`
+        );
+      } else {
+        outputProducers.set(derivation.output, id);
+      }
+    }
+
+    requireRef('activity', derivation.activity, path);
+
+    const inputEntities = new Set<string>();
+    for (let j = 0; j < derivation.inputs.length; j++) {
+      const input = derivation.inputs[j]!;
+      const inputPath = `${path}.inputs[${j}]`;
+      requireRef('entity', input.entity, inputPath);
+      if (input.entity === derivation.output) {
+        issues.push(`${inputPath}: output cannot be its own input`);
+      }
+      if (inputEntities.has(input.entity)) {
+        issues.push(`${inputPath}: duplicate input entity "${input.entity}"`);
+      }
+      inputEntities.add(input.entity);
+      requireConfidenceRefs(input.confidence, `${inputPath}.confidence`);
+    }
+
+    requireConfidenceRefs(derivation.confidence, `${path}.confidence`);
+  }
+
+  // Document outputs resolve.
+  const seenOutputs = new Set<string>();
+  for (let i = 0; i < doc.outputs.length; i++) {
+    const outputId = doc.outputs[i]!;
+    if (!entityIds.has(outputId)) {
+      issues.push(`outputs[${i}]: entity "${outputId}" does not exist`);
+    }
+    if (seenOutputs.has(outputId)) {
+      issues.push(`outputs[${i}]: duplicate entity "${outputId}"`);
+    }
+    seenOutputs.add(outputId);
+  }
+
+  // Entity attributions, sign-offs, and confidence sources.
+  for (const [entityId, entity] of Object.entries(doc.entities)) {
+    const entityPath = `entities.${entityId}`;
+    entity.attributions?.forEach((attribution, i) => {
+      requireRef('agent', attribution.agent, `${entityPath}.attributions[${i}]`);
+    });
+    entity.approvals?.forEach((approval, i) => {
+      requireRef('agent', approval.agent, `${entityPath}.approvals[${i}]`);
+    });
+    entity.confidence?.forEach((assertion, i) => {
+      requireConfidenceRefs(assertion, `${entityPath}.confidence[${i}]`);
+    });
+  }
+
+  // Activity associations + timestamp ordering. RFC 3339 syntax is a shape
+  // check; this pass still rejects NaN instants if a document bypassed parse.
+  for (const [activityId, activity] of Object.entries(doc.activities)) {
+    activity.associations?.forEach((association, i) => {
+      requireRef('agent', association.agent, `activities.${activityId}.associations[${i}]`);
+    });
+
+    const { startedAt, endedAt } = activity;
+    if (startedAt !== undefined && endedAt !== undefined) {
+      const startedMs = Date.parse(startedAt);
+      const endedMs = Date.parse(endedAt);
+      if (Number.isNaN(startedMs) || Number.isNaN(endedMs)) {
+        issues.push(
+          `activities.${activityId}: startedAt or endedAt is not a valid RFC 3339 timestamp`
+        );
+      } else if (endedMs < startedMs) {
+        issues.push(`activities.${activityId}: endedAt precedes startedAt`);
+      }
+    }
+  }
+
+  // Relations: resolve + collection membership correctness.
+  for (let i = 0; i < doc.relations.length; i++) {
+    const relation = doc.relations[i]!;
+    const path = `relations[${i}]`;
+    requireRef('entity', relation.source, `${path}.source`);
+    requireRef('entity', relation.target, `${path}.target`);
+    if (
+      relation.type === 'member_of' &&
+      entityIds.has(relation.target) &&
+      doc.entities[relation.target]?.kind !== 'collection'
+    ) {
+      issues.push(`${path}: member_of target "${relation.target}" must be a collection entity`);
+    }
+  }
+
+  // Entity dependency graph via derivations must be acyclic.
+  const dependents = new Map<string, string[]>();
+  for (const id of entityIds) dependents.set(id, []);
+  for (const derivation of doc.derivations) {
+    if (!entityIds.has(derivation.output)) continue;
+    for (const input of derivation.inputs) {
+      if (!entityIds.has(input.entity)) continue;
+      dependents.get(input.entity)!.push(derivation.output);
+    }
+  }
+
+  const visiting = new Set<string>();
+  const visited = new Set<string>();
+  const cyclePath: string[] = [];
+
+  const visit = (id: string): boolean => {
+    if (visited.has(id)) return false;
+    if (visiting.has(id)) {
+      cyclePath.push(id);
+      return true;
+    }
+    visiting.add(id);
+    for (const next of dependents.get(id) ?? []) {
+      if (visit(next)) {
+        cyclePath.push(id);
+        return true;
+      }
+    }
+    visiting.delete(id);
+    visited.add(id);
+    return false;
+  };
+
+  for (const id of entityIds) {
+    if (visit(id)) {
+      const cycle = [...new Set(cyclePath.reverse())].join(' → ');
+      issues.push(`entity dependency cycle detected: ${cycle}`);
+      break;
+    }
+  }
+
+  if (issues.length > 0) {
+    throw new LineageValidationError(issues);
+  }
+}