coaia-visualizer 1.6.1 → 1.6.3

package/lib/jsonl-preservation.ts

@@ -0,0 +1,405 @@
+import type {
+  EntityRecord,
+  MetadataWarning,
+  ParsedData,
+  RawJSONLRecord,
+  RecordProjections,
+  RelationRecord,
+} from "./types"
+
+const UPSTREAM_ISSUE = "https://github.com/avadisabelle/coaia-narrative/issues/35"
+
+const ENTITY_TOP_LEVEL_KEYS = new Set(["type", "name", "entityType", "observations", "metadata"])
+const RELATION_TOP_LEVEL_KEYS = new Set(["type", "from", "to", "relationType", "metadata"])
+const NARRATIVE_BEAT_TOP_LEVEL_KEYS = new Set([
+  ...ENTITY_TOP_LEVEL_KEYS,
+  "narrative",
+  "relational_alignment",
+  "four_directions",
+])
+const MUTABLE_METADATA_KEYS = new Set([
+  "dueDate",
+  "chartId",
+  "phase",
+  "completionStatus",
+  "completedAt",
+  "parentChart",
+  "parentActionStep",
+  "level",
+  "createdAt",
+  "updatedAt",
+  "act",
+  "type_dramatic",
+  "universes",
+  "timestamp",
+  "elementsOfPerformance",
+  "mmotEvaluations",
+  "relationalAlignment",
+  "fourDirections",
+  "narrative",
+  "isTelescopedChart",
+  "telescopedChartId",
+])
+
+function isObject(value: unknown): value is Record<string, any> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function cloneJson<T>(value: T): T {
+  return value === undefined ? value : JSON.parse(JSON.stringify(value))
+}
+
+function deepEqual(left: unknown, right: unknown): boolean {
+  return JSON.stringify(left) === JSON.stringify(right)
+}
+
+function removeUndefined<T extends Record<string, any>>(record: T): T {
+  for (const key of Object.keys(record)) {
+    if (record[key] === undefined) {
+      delete record[key]
+    }
+  }
+  return record
+}
+
+function relationKey(relation: Pick<RelationRecord, "from" | "to" | "relationType">): string {
+  return `${relation.from}\u0000${relation.to}\u0000${relation.relationType}`
+}
+
+export function isEntityLikeRecord(record: RawJSONLRecord): boolean {
+  return (record.type === "entity" || record.type === "narrative_beat") && typeof record.name === "string"
+}
+
+export function isRelationLikeRecord(record: RawJSONLRecord): boolean {
+  return (
+    record.type === "relation" &&
+    typeof record.from === "string" &&
+    typeof record.to === "string" &&
+    typeof record.relationType === "string"
+  )
+}
+
+export function normalizeJsonlRecord(record: RawJSONLRecord): EntityRecord | RelationRecord | null {
+  if (record.type === "entity" && typeof record.name === "string") {
+    return {
+      ...record,
+      type: "entity",
+      observations: Array.isArray(record.observations) ? record.observations : [],
+      metadata: isObject(record.metadata) ? record.metadata : {},
+    } as EntityRecord
+  }
+
+  if (record.type === "narrative_beat" && typeof record.name === "string") {
+    const metadata = {
+      ...(isObject(record.metadata) ? record.metadata : {}),
+      narrative: isObject(record.metadata) && record.metadata.narrative !== undefined
+        ? record.metadata.narrative
+        : record.narrative,
+      relationalAlignment: isObject(record.metadata) && record.metadata.relationalAlignment !== undefined
+        ? record.metadata.relationalAlignment
+        : record.relational_alignment,
+      fourDirections: isObject(record.metadata) && record.metadata.fourDirections !== undefined
+        ? record.metadata.fourDirections
+        : record.four_directions,
+    }
+
+    return removeUndefined({
+      ...record,
+      type: "entity",
+      name: record.name as string,
+      entityType: "narrative_beat",
+      observations: Array.isArray(record.observations) ? record.observations : [],
+      metadata: removeUndefined(metadata),
+    }) as EntityRecord
+  }
+
+  if (isRelationLikeRecord(record)) {
+    return {
+      ...record,
+      type: "relation",
+      metadata: isObject(record.metadata) ? record.metadata : {},
+    } as RelationRecord
+  }
+
+  return null
+}
+
+function mergePreservedMetadata(originalMetadata: unknown, currentMetadata: unknown): Record<string, any> {
+  const merged = {
+    ...(isObject(currentMetadata) ? currentMetadata : {}),
+  }
+
+  if (isObject(originalMetadata)) {
+    for (const [key, value] of Object.entries(originalMetadata)) {
+      if (!(key in merged) && (!MUTABLE_METADATA_KEYS.has(key) || key === "github")) {
+        merged[key] = cloneJson(value)
+      }
+    }
+  }
+
+  return merged
+}
+
+function serializeEntity(entity: EntityRecord, original?: RawJSONLRecord): RawJSONLRecord {
+  const metadata = mergePreservedMetadata(original?.metadata, entity.metadata)
+
+  if (original?.type === "narrative_beat") {
+    const record: RawJSONLRecord = {
+      ...original,
+      type: "narrative_beat",
+      name: entity.name,
+      observations: entity.observations,
+      metadata,
+    }
+
+    if ("entityType" in original) {
+      record.entityType = entity.entityType
+    }
+    if ("narrative" in original && entity.metadata?.narrative !== undefined) {
+      record.narrative = cloneJson(entity.metadata.narrative)
+    }
+    if ("relational_alignment" in original && entity.metadata?.relationalAlignment !== undefined) {
+      record.relational_alignment = cloneJson(entity.metadata.relationalAlignment)
+    }
+    if ("four_directions" in original && entity.metadata?.fourDirections !== undefined) {
+      record.four_directions = cloneJson(entity.metadata.fourDirections)
+    }
+
+    return removeUndefined(record)
+  }
+
+  return removeUndefined({
+    ...original,
+    ...entity,
+    type: "entity",
+    metadata,
+  })
+}
+
+function serializeRelation(relation: RelationRecord, original?: RawJSONLRecord): RawJSONLRecord {
+  return removeUndefined({
+    ...original,
+    ...relation,
+    type: "relation",
+    metadata: mergePreservedMetadata(original?.metadata, relation.metadata),
+  })
+}
+
+function findRecord(records: RawJSONLRecord[], key: string): RawJSONLRecord | undefined {
+  return records.find((record) => {
+    if (isEntityLikeRecord(record) && record.name === key) return true
+    if (isRelationLikeRecord(record)) return relationKey(record as RelationRecord) === key
+    return false
+  })
+}
+
+function validateMetadata(label: string, beforeMetadata: unknown, afterMetadata: unknown, failures: string[]): void {
+  if (!isObject(beforeMetadata)) return
+  if (!isObject(afterMetadata)) {
+    failures.push(`${label}: metadata object missing after write`)
+    return
+  }
+
+  for (const [key, value] of Object.entries(beforeMetadata)) {
+    if (!(key in afterMetadata)) {
+      failures.push(`${label}: metadata.${key} missing after write`)
+      continue
+    }
+
+    if ((key === "github" || !MUTABLE_METADATA_KEYS.has(key)) && !deepEqual(value, afterMetadata[key])) {
+      failures.push(`${label}: metadata.${key} changed unexpectedly`)
+    }
+  }
+}
+
+function validateRecordPreservation(
+  beforeRecords: RawJSONLRecord[],
+  afterRecords: RawJSONLRecord[],
+  entities: Map<string, EntityRecord>,
+  relations: RelationRecord[],
+): void {
+  const relationKeys = new Set(relations.map((relation) => relationKey(relation)))
+  const failures: string[] = []
+
+  for (const before of beforeRecords) {
+    if (isEntityLikeRecord(before)) {
+      const name = before.name as string
+      if (!entities.has(name)) continue
+
+      const after = findRecord(afterRecords, name)
+      if (!after) {
+        failures.push(`${name}: record missing after write`)
+        continue
+      }
+
+      if (before.type === "narrative_beat" && after.type !== "narrative_beat") {
+        failures.push(`${name}: legacy narrative_beat record type was flattened`)
+      }
+
+      const knownKeys = before.type === "narrative_beat" ? NARRATIVE_BEAT_TOP_LEVEL_KEYS : ENTITY_TOP_LEVEL_KEYS
+      for (const key of Object.keys(before)) {
+        if (!knownKeys.has(key) && !deepEqual(before[key], after[key])) {
+          failures.push(`${name}: top-level extension field "${key}" was not preserved`)
+        }
+      }
+      validateMetadata(name, before.metadata, after.metadata, failures)
+    }
+
+    if (isRelationLikeRecord(before)) {
+      const key = relationKey(before as RelationRecord)
+      if (!relationKeys.has(key)) continue
+
+      const after = findRecord(afterRecords, key)
+      if (!after) {
+        failures.push(`${key}: relation missing after write`)
+        continue
+      }
+
+      for (const topLevelKey of Object.keys(before)) {
+        if (!RELATION_TOP_LEVEL_KEYS.has(topLevelKey) && !deepEqual(before[topLevelKey], after[topLevelKey])) {
+          failures.push(`${key}: relation extension field "${topLevelKey}" was not preserved`)
+        }
+      }
+      validateMetadata(key, before.metadata, after.metadata, failures)
+    }
+  }
+
+  if (failures.length > 0) {
+    throw new Error(`COAIA JSONL metadata preservation failed:\n- ${failures.join("\n- ")}`)
+  }
+}
+
+export function buildRecordProjections(entities: Map<string, EntityRecord>): RecordProjections {
+  const allEntities = Array.from(entities.values())
+  return {
+    chartRecords: allEntities.filter((entity) => entity.entityType === "structural_tension_chart"),
+    actionStepRecords: allEntities.filter((entity) => entity.entityType === "action_step" || entity.metadata?.isTelescopedChart),
+    narrativeBeatRecords: allEntities.filter((entity) => entity.entityType === "narrative_beat"),
+  }
+}
+
+export function buildMetadataWarnings(records: RawJSONLRecord[]): MetadataWarning[] {
+  const warnings: MetadataWarning[] = []
+
+  for (const record of records) {
+    if (!isEntityLikeRecord(record) && !isRelationLikeRecord(record)) continue
+
+    const recordName = record.name || [record.from, record.to, record.relationType].filter(Boolean).join(" -> ")
+    const metadata = record.metadata
+
+    if (!isObject(metadata)) {
+      warnings.push({
+        severity: "warning",
+        recordName,
+        recordType: record.type,
+        message: "Record has no metadata object; provenance and extension fields may already be flattened.",
+        upstreamIssue: UPSTREAM_ISSUE,
+      })
+      continue
+    }
+
+    if (record.github && !metadata.github) {
+      warnings.push({
+        severity: "warning",
+        recordName,
+        recordType: record.type,
+        message: "GitHub provenance exists outside metadata.github; this shape is suspicious for preservation-aware tools.",
+        upstreamIssue: UPSTREAM_ISSUE,
+      })
+    }
+
+    const normalized = normalizeJsonlRecord(record)
+    if (normalized && "entityType" in normalized) {
+      if (
+        ["structural_tension_chart", "action_step", "desired_outcome", "current_reality", "narrative_beat"].includes(
+          normalized.entityType,
+        ) &&
+        !normalized.metadata?.chartId
+      ) {
+        warnings.push({
+          severity: "warning",
+          recordName: normalized.name,
+          recordType: normalized.entityType,
+          message: "COAIA entity is missing metadata.chartId, so chart/action projection may be incomplete.",
+          upstreamIssue: UPSTREAM_ISSUE,
+        })
+      }
+
+      if (
+        normalized.entityType === "narrative_beat" &&
+        !normalized.metadata?.narrative &&
+        !record.narrative &&
+        normalized.observations.length <= 1
+      ) {
+        warnings.push({
+          severity: "warning",
+          recordName: normalized.name,
+          recordType: normalized.entityType,
+          message: "Narrative beat appears flattened: no narrative metadata or rich beat payload is visible.",
+          upstreamIssue: UPSTREAM_ISSUE,
+        })
+      }
+    }
+
+    if (record.type === "narrative_beat") {
+      warnings.push({
+        severity: "warning",
+        recordName,
+        recordType: "narrative_beat",
+        message: "Legacy narrative_beat shape detected. It is rendered separately and preserved on save.",
+        upstreamIssue: UPSTREAM_ISSUE,
+      })
+    }
+  }
+
+  return warnings
+}
+
+export function serializeParsedData(data: Pick<ParsedData, "entities" | "relations" | "rawRecords">): string {
+  const rawRecords = data.rawRecords || []
+  const emittedEntities = new Set<string>()
+  const emittedRelations = new Set<string>()
+  const records: RawJSONLRecord[] = []
+
+  for (const original of rawRecords) {
+    if (isEntityLikeRecord(original)) {
+      const entity = data.entities.get(original.name)
+      if (entity) {
+        records.push(serializeEntity(entity, original))
+        emittedEntities.add(entity.name)
+      }
+      continue
+    }
+
+    if (isRelationLikeRecord(original)) {
+      const key = relationKey(original as RelationRecord)
+      const relation = data.relations.find((candidate) => relationKey(candidate) === key)
+      if (relation) {
+        records.push(serializeRelation(relation, original))
+        emittedRelations.add(key)
+      }
+      continue
+    }
+
+    records.push(original)
+  }
+
+  for (const entity of data.entities.values()) {
+    if (!emittedEntities.has(entity.name)) {
+      records.push(serializeEntity(entity))
+    }
+  }
+
+  for (const relation of data.relations) {
+    const key = relationKey(relation)
+    if (!emittedRelations.has(key)) {
+      records.push(serializeRelation(relation))
+    }
+  }
+
+  validateRecordPreservation(rawRecords, records, data.entities, data.relations)
+
+  return `${records.map((record) => JSON.stringify(record)).join("\n")}\n`
+}
+
+export { UPSTREAM_ISSUE }

package/lib/types.ts

@@ -6,6 +6,7 @@ export interface EntityRecord {
   entityType: "structural_tension_chart" | "desired_outcome" | "current_reality" | "action_step" | "narrative_beat"
   observations: string[]
   metadata: Record<string, any>
+  [key: string]: any
 }
 
 export interface RelationRecord {
@@ -14,9 +15,34 @@ export interface RelationRecord {
   to: string
   relationType: "contains" | "creates_tension_with" | "advances_toward" | "documents"
   metadata: Record<string, any>
+  [key: string]: any
 }
 
 export type JSONLRecord = EntityRecord | RelationRecord
+export type RawJSONLRecord = Record<string, any>
+
+export interface MetadataWarning {
+  severity: "warning" | "error"
+  recordName?: string
+  recordType?: string
+  message: string
+  upstreamIssue?: string
+}
+
+export interface RecordProjections {
+  chartRecords: EntityRecord[]
+  actionStepRecords: EntityRecord[]
+  narrativeBeatRecords: EntityRecord[]
+}
+
+export interface RiseSpecProjection {
+  chartId: string
+  desiredOutcome: string
+  currentReality: string[]
+  naturalProgression: string[]
+  remainingBlockers: string[]
+  nextOperations: string[]
+}
 
 export interface Chart {
   id: string
@@ -36,4 +62,8 @@ export interface ParsedData {
   entities: Map<string, EntityRecord>
   relations: RelationRecord[]
   rootCharts: Chart[]
+  rawRecords?: RawJSONLRecord[]
+  projections?: RecordProjections
+  preservationWarnings?: MetadataWarning[]
+  riseSpecs?: RiseSpecProjection[]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coaia-visualizer",
-  "version": "1.6.1",
+  "version": "1.6.3",
   "private": false,
   "type": "module",
   "main": "./index.tsx",
@@ -18,6 +18,35 @@
   "bin": {
     "coaia-visualizer": "dist/cli.js"
   },
+  "files": [
+    "app/**/*",
+    "components/**/*",
+    "hooks/**/*",
+    "lib/**/*",
+    "styles/**/*",
+    "public/**/*",
+    "dist/**/*",
+    "mcp/dist/**/*",
+    "mcp/src/**/*",
+    "mcp/package.json",
+    "mcp/package-lock.json",
+    "mcp/README.md",
+    "mcp/Dockerfile",
+    "mcp/tsconfig.json",
+    "index.tsx",
+    "cli.ts",
+    "skill.ts",
+    "components.json",
+    "mcp-config.json",
+    "next-env.d.ts",
+    "next.config.mjs",
+    "postcss.config.mjs",
+    "tsconfig.json",
+    "README.md",
+    "README_TELESCOPED_NAVIGATION.md",
+    "KINSHIP.md",
+    "NAMING.md"
+  ],
   "scripts": {
     "build": "next build",
     "build:cli": "tsc cli.ts --outDir dist --module esnext --target es2022 --moduleResolution bundler --esModuleInterop --skipLibCheck",
@@ -26,6 +55,7 @@
     "prepare": "npm run build:cli",
     "start": "next start",
     "test": "playwright test",
+    "test:rich-jsonl": "playwright test test-scripts/rich-jsonl-preservation.spec.ts",
     "test:global-cli": "playwright test test-scripts/test-global-cli.spec.ts",
     "docker:build": "docker build -t jgwill/coaia:visualizer .",
     "docker:push": "docker push jgwill/coaia:visualizer",

package/.dockerignore

@@ -1,9 +0,0 @@
-node_modules
-dist
-.next
-.git
-.env.local
-.env*.local
-test-results
-*.log
-.DS_Store