@aiaiai-pt/frankctl 0.5.0 → 0.5.1

package/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to `@aiaiai-pt/frankctl` are documented here. This project
 adheres to [Semantic Versioning](https://semver.org/).
 
+## 0.5.1
+
+- Make `pipelines apply --wait` consume the precision-safe ontology
+  `snapshot_id_exact`, retaining string-valued `snapshot_id` only as an old-API
+  fallback and continuing to reject numeric-only evidence.
+
 ## 0.5.0
 
 - Make `Source.spec.streams` declarative desired state: create or patch named

package/dist/index.js

@@ -244,7 +244,7 @@ ${n}`}function qt(t){let e=J({profile:t.profile,apiUrl:t.apiUrl}),r={enabled:pro
 `),{ok:!0,created:await Lb(t,e,{})}):{ok:!1}}function eP(t){let e=t.filter(a=>a.kind==="BackingDataset"),r=new Map;for(let a of e)a.backing_dataset_identity&&r.set(Le(a.backing_dataset_identity),a);let i=[],n=new Set,o=new Set,s=a=>{let c=a.backing_dataset_identity,l=c?Le(c):a.id;if(n.has(l))throw new w(`BackingDataset dependency cycle reached ${a.name} (${l})`);if(!o.has(l)){n.add(l);for(let u of a.backing_dataset_dependencies??[]){let d=r.get(Le(u));d&&s(d)}n.delete(l),o.add(l),i.push(a)}};for(let a of e)s(a);return i}async function tP(t){let e=new Map,r=1;for(;;){let i=await t.get("/api/v1/backing-datasets",{page:r,page_size:100});for(let n of i.items??[]){let o=String(n.entity_type_id??"").trim();o&&e.set(Le({entityTypeId:o,ontologyTenantId:n.ontology_tenant_id??null}),n)}if(!i.has_more)return e;r+=1}}var nP=new Set(["completed","partial","failed","cancelled","terminated","timed_out","error"]),rP=new Set(["completed","failed","cancelled"]);function It(t){return String(t??"").trim().toLowerCase()}function cm(t,e){if(typeof t!="string"||!/^-?\d+$/.test(t))throw new w(`${e} must return snapshot_id as an exact decimal string; numeric JSON snapshot IDs are rejected because JavaScript may already have rounded them.`);return t}function iP(t,e,r){if(e.length===0)throw new w(`Pipeline/${t} has no hydrated transforms; refusing to report --wait success.`);if(e.length>1&&!Array.isArray(r?.edges))throw new w(`Pipeline/${t} has multiple hydrated transforms but no persisted DAG edges; refusing alphabetical execution.`);let i=new Map;for(let u of e){let d=String(u.step_name??"").trim();if(!d)throw new w(`Pipeline/${t} returned a hydrated step without step_name.`);if(i.has(d))throw new w(`Pipeline/${t} returned duplicate step '${d}'.`);let p=String(u.transform_id??"").trim();if(!p)throw new w(`Pipeline/${t} step '${d}' has no hydrated transform.`);i.set(d,{stepName:d,transformId:p})}let n=[];for(let u of r?.steps??[]){let d=String(u.step_name??u.name??"").trim();if(!d)throw new w(`Pipeline/${t} DAG contains a step without a name.`);if(!i.has(d))throw new w(`Pipeline/${t} DAG step '${d}' has no hydrated transform.`);if(n.includes(d))throw new w(`Pipeline/${t} DAG contains duplicate step '${d}'.`);n.push(d)}if(Array.isArray(r?.steps)){let u=[...i.keys()];if(n.length!==u.length||u.some(d=>!n.includes(d)))throw new w(`Pipeline/${t} DAG step set (${n.join(", ")||"empty"}) does not match hydrated step set (${u.join(", ")||"empty"}); no transform was triggered.`)}else for(let u of i.keys())n.push(u);let o=new Map(n.map((u,d)=>[u,d])),s=new Map(n.map(u=>[u,new Set])),a=new Map(n.map(u=>[u,0]));for(let u of r?.edges??[]){let d=String(u.from_step??u.from_step_name??"").trim(),p=String(u.to_step??u.to_step_name??"").trim();if(!d||!p)throw new w(`Pipeline/${t} DAG contains an edge without from_step/to_step.`);if(!i.has(d)||!i.has(p))throw new w(`Pipeline/${t} DAG edge '${d}' \u2192 '${p}' references a step without a hydrated transform.`);let g=s.get(d);g.has(p)||(g.add(p),a.set(p,(a.get(p)??0)+1))}let c=n.filter(u=>a.get(u)===0).sort((u,d)=>o.get(u)-o.get(d)),l=[];for(;c.length>0;){let u=c.shift();l.push(u);for(let d of s.get(u)??[]){let p=(a.get(d)??0)-1;a.set(d,p),p===0&&(c.push(d),c.sort((g,f)=>o.get(g)-o.get(f)))}}if(l.length!==i.size)throw new w(`Pipeline/${t} DAG contains a cycle; no transform was triggered.`);return l.map(u=>i.get(u))}function oP(t,e){if(t.length===1)return t[0];let r=new Set((e?.edges??[]).map(n=>String(n.from_step??n.from_step_name??"").trim())),i=t.filter(n=>!r.has(n.stepName));return i.length===1?i[0]:void 0}async function sP(t,e,r){try{let i=await t.get(`/api/v1/sources/sync-runs/${encodeURIComponent(e)}/details`);if(i.workflow_id!==e)throw new w(`Source workflow correlation mismatch: requested '${e}', received '${i.workflow_id}'.`);if(!i.sync_run_id||i.source_id!==r)throw new w(`Source workflow correlation mismatch: workflow_id=${e} did not return the expected sync_run_id/source_id=${r}.`);return i}catch(i){if(i instanceof F&&i.status===404)return{workflow_id:e,status:"pending"};throw i}}async function aP(t,e,r,i){let n=await t.get(`/api/v1/sources/${e}/iceberg-refs`);if(n.source_id!==e)throw new w(`Source/${e} Iceberg observation returned source_id=${n.source_id}.`);if(!n.available)throw new w(`Source/${e} Iceberg refs unavailable: ${n.reason??"no reason returned"}.`);let o=new Map(n.tables.map(c=>[c.stream_name,c])),s=r.length>0?r:n.tables.map(c=>c.stream_name);if(r.length===0&&n.tables.length!==i)throw new w(`Source/${e} completed ${i} stream(s), but Iceberg refs returned ${n.tables.length} Bronze table(s).`);if(s.length===0&&i>0)throw new w(`Source/${e} completed ${i} stream(s), but Iceberg refs returned no Bronze tables.`);let a=[];for(let c of s){let l=o.get(c);if(!l)throw new w(`Source/${e} enabled stream '${c}' has no returned Bronze table.`);if(!l.available)throw new w(`Source/${e} Bronze table for stream '${c}' is unavailable: ${l.reason??"no reason returned"}.`);let u=(l.refs??[]).find(p=>p.name==="main");if(!u||u.snapshot_id==null)throw new w(`Source/${e} Bronze table '${l.table_path??c}' has no main snapshot.`);let d=cm(u.snapshot_id,`Source/${e} Bronze table '${l.table_path??c}'`);a.push({stream_name:c,table_path:l.table_path??c,ref:"main",snapshot_id:d,total_records:u.total_records,evidence_scope:"observed_after_terminal_not_workflow_correlated"})}return a}async function cP(t,e,r){let i=await ye({fetch:()=>t.get("/api/v1/runs",{dagster_run_id:e,page_size:2}),isTerminal:a=>a.items.length>0,intervalMs:Math.min(1e3,r),timeoutMs:r});if(i.timedOut||i.value.items.length===0)throw new w(`Timed out waiting for TransformRun correlated to dagster_run_id=${e}.`);if(i.value.items.length!==1)throw new w(`Expected one TransformRun for dagster_run_id=${e}, received ${i.value.items.length}.`);let n=i.value.items[0];if(n.dagster_run_id!==e||!n.id)throw new w(`TransformRun correlation mismatch for dagster_run_id=${e}.`);let o=n.id,s=await ye({fetch:async()=>{let a=await t.get(`/api/v1/runs/${encodeURIComponent(o)}`);if(a.id!==o||a.dagster_run_id!==e)throw new w(`TransformRun correlation changed while polling transform_run_id=${o}.`);return a},isTerminal:a=>rP.has(It(a.status)),intervalMs:Math.min(5e3,r),timeoutMs:r});if(s.timedOut)throw new w(`Timed out waiting for transform_run_id=${o} (dagster_run_id=${e}).`);return s.value}function lP(t){if(!(t instanceof F)||t.status!==409)return{matches:!1};let e=t.body?.detail;if(e&&typeof e=="object"){let r=e,n=It(r.code??r.error??r.reason).replaceAll("-","_")==="sync_already_running",o=String(r.sync_run_id??"").trim(),s=String(r.workflow_id??"").trim();return{matches:n,...n&&o?{syncRunId:o}:{},...n&&s?{workflowId:s}:{}}}return{matches:!1}}async function uP(t,e,r,i,n){let o=await ye({fetch:async()=>{let s=await t.get(`/api/v1/backing-datasets/${e}/sync-history/${encodeURIComponent(r)}`);if(s.id!==r||s.backing_dataset_id!==e||s.workflow_id!==i)throw new w(`Ontology sync correlation mismatch for sync_run_id=${r} workflow_id=${i}.`);return s},isTerminal:s=>s.is_terminal===!0||["synced","error","skipped"].includes(It(s.status)),intervalMs:Math.min(6e3,n),timeoutMs:n});if(o.timedOut)throw new w(`Timed out waiting for BackingDataset/${e} sync_run_id=${r} workflow_id=${i}.`);return o.value}async function dP(t,e,r){let i=f=>process.stderr.write(f+`
 `),n=!0,o={ok:!1,sources:[],transforms:[],backing_datasets:[]},s=new Set(e.filter(f=>f.kind==="Pipeline").map(f=>f.id)),a=eP(e),c=new Map;for(let f of a)f.backing_dataset_identity&&c.set(Le(f.backing_dataset_identity),f);let l=new Set,u,d=new Map,p=new Map,g=()=>(o.ok=n,i(n?`
 wait: \u2713 all declared stages reached correlated terminal success; ontology entity readback remains separate proof.`:`
-wait: \u2717 one or more stages failed (see above).`),o);for(let f of e.filter(m=>m.kind==="Source")){i(`wait: syncing Source/${f.name} \u2192 bronze...`);let m,v;try{let P=await t.get(`/api/v1/sources/${f.id}`),z=Array.isArray(P.streams)?P.streams:[],b=new Map(z.map(V=>[String(V.name??"").trim(),V])),x=f.declared_stream_names?[...f.declared_stream_names]:z.filter(V=>V.is_enabled!==!1).map(V=>String(V.name??"").trim()).filter(Boolean);if(f.declared_stream_names){if(x.length===0)throw new w(`Source/${f.name} declares no enabled streams; --wait will not fall back to unmanaged streams.`);for(let V of x){let fe=b.get(V);if(!fe||fe.is_enabled===!1)throw new w(`Source/${f.name} declared enabled stream '${V}', but the applied Source does not expose it as enabled.`)}}P.status==="draft"&&await t.patch(`/api/v1/sources/${f.id}`,{status:"ready"});let D=await t.post(`/api/v1/sources/${f.id}/sync`,f.declared_stream_names?{stream_names:x}:{});if(m=String(D.workflow_id??"").trim(),!m)throw new w(`Source/${f.name} sync response omitted workflow_id; cannot correlate --wait.`);let C=m,A=await ye({fetch:()=>sP(t,C,f.id),isTerminal:V=>nP.has(It(V.status)),intervalMs:Math.min(5e3,r),timeoutMs:r});v=A.value;let O=It(A.value.status),M=A.timedOut?"timeout":O;if(A.timedOut||O!=="completed")o.sources.push({name:f.name,id:f.id,workflow_id:m,...A.value.sync_run_id?{sync_run_id:A.value.sync_run_id}:{},...A.value.workflow_run_id?{workflow_run_id:A.value.workflow_run_id}:{},status:M,...A.value.error_message?{error:A.value.error_message}:{},...A.value.temporal_ui_url?{temporal_ui_url:A.value.temporal_ui_url}:{}}),i(`  \u2717 Source/${f.name}: ${M||"error"} workflow_id=${m}`),n=!1;else{if(A.value.records_synced==null||A.value.total_streams==null||A.value.successful_streams==null||A.value.failed_streams==null)throw new w(`Source/${f.name} workflow_id=${m} completed without persisted stream/row metrics.`);if(A.value.total_streams!==D.streams_count)throw new w(`Source/${f.name} workflow_id=${m} persisted total_streams=${A.value.total_streams}, but the trigger declared streams_count=${D.streams_count}.`);if(x.length>0&&x.length!==D.streams_count)throw new w(`Source/${f.name} trigger declared streams_count=${D.streams_count}, but source details returned ${x.length} enabled stream(s).`);if(A.value.failed_streams!==0||A.value.successful_streams!==A.value.total_streams)throw new w(`Source/${f.name} workflow_id=${m} completed with ${A.value.successful_streams}/${A.value.total_streams} successful streams and ${A.value.failed_streams} failures.`);let V=await aP(t,f.id,x,D.streams_count);o.sources.push({name:f.name,id:f.id,workflow_id:m,sync_run_id:A.value.sync_run_id,...A.value.workflow_run_id?{workflow_run_id:A.value.workflow_run_id}:{},status:O,row_count:A.value.records_synced,persisted_metrics:{total_streams:A.value.total_streams,successful_streams:A.value.successful_streams,failed_streams:A.value.failed_streams,records_synced:A.value.records_synced,bytes_synced:A.value.bytes_synced,batches_completed:A.value.batches_completed,total_batches_estimated:A.value.total_batches_estimated,records_written_so_far:A.value.records_written_so_far,failed_batches:A.value.failed_batches},observed_bronze_refs:V,...A.value.temporal_ui_url?{temporal_ui_url:A.value.temporal_ui_url}:{}}),i(`  \u2713 bronze synced (${f.name}) workflow_id=${m} sync_run_id=${A.value.sync_run_id} rows=${A.value.records_synced} observed_main_refs=${V.map(fe=>`${fe.table_path}@${fe.snapshot_id}`).join(",")}`+(A.value.temporal_ui_url?` temporal=${A.value.temporal_ui_url}`:""))}}catch(P){let z=P.message;o.sources.push({name:f.name,id:f.id,...m?{workflow_id:m}:{},...v?.sync_run_id?{sync_run_id:v.sync_run_id}:{},...v?.workflow_run_id?{workflow_run_id:v.workflow_run_id}:{},status:"error",error:z}),i(`  \u2717 Source/${f.name}: ${z}`),n=!1}}if(!n)return i("wait: source stage failed; downstream transforms were not triggered."),g();for(let f of e.filter(m=>m.kind==="Pipeline")){i(`wait: materializing Pipeline/${f.name} \u2192 silver...`);try{let m=await t.get(`/api/v1/pipelines/${f.id}`,{include_version:"true"}),v=m.current_version?.steps??[],P=iP(f.name,v,m.current_version?.dag_spec),z=oP(P,m.current_version?.dag_spec);for(let b of P){let x,D,C="error";try{let A=await t.post(`/api/v1/transforms/${b.transformId}/materialize`,{wait_for_completion:!1,timeout:Math.min(3600,Math.max(10,Math.ceil(r/1e3)))});if(x=String(A.dagster_run_id??"").trim(),!A.success||!x)throw new w(`Transform '${b.stepName}' was not started: ${A.error_message??"materialize response omitted dagster_run_id"}.`);let O=await cP(t,x,r);if(D=O.id,C=It(O.status),C!=="completed")throw new w(`Transform '${b.stepName}' failed: ${C} dagster_run_id=${x} transform_run_id=${O.id} ${O.error_message??""}`);if(O.output_snapshot_id==null)throw new w(`Transform '${b.stepName}' completed without output_snapshot_id (dagster_run_id=${x}, transform_run_id=${O.id}).`);let M=cm(O.output_snapshot_id,`Transform '${b.stepName}' (dagster_run_id=${x}, transform_run_id=${O.id})`),V=O.rows_affected??O.output_row_count;if(V==null)throw new w(`Transform '${b.stepName}' completed without an observable row count (dagster_run_id=${x}, transform_run_id=${O.id}).`);o.transforms.push({pipeline_name:f.name,pipeline_id:f.id,step_name:b.stepName,transform_id:b.transformId,dagster_run_id:x,transform_run_id:O.id,status:C,row_count:V,snapshot_id:M}),d.set(b.transformId,{snapshotId:M,transformId:b.transformId,transformRunId:O.id,pipelineId:f.id}),i(`  \u2713 silver materialized step=${b.stepName} dagster_run_id=${x} transform_run_id=${O.id} snapshot=${M} rows=${V}`)}catch(A){let O=A.message;throw o.transforms.push({pipeline_name:f.name,pipeline_id:f.id,step_name:b.stepName,transform_id:b.transformId,...x?{dagster_run_id:x}:{},...D?{transform_run_id:D}:{},status:C,error:O}),A}}if(z){let b=d.get(z.transformId);b&&p.set(f.id,b)}}catch(m){i(`  \u2717 materialize ${f.name}: ${m.message}`),n=!1}}if(!n)return i("wait: transform stage failed; ontology sync was not triggered."),g();for(let f of a){i(`wait: syncing BackingDataset/${f.name} \u2192 ontology...`);let m,v,P=!1,z="error";try{for(let O of f.backing_dataset_dependencies??[]){let M=Le(O);if(c.get(M)){if(!l.has(M))throw new w(`BackingDataset/${f.name} dependency '${M}' did not reach terminal success in this invocation`);continue}u??=await tP(t);let fe=u.get(M);if(!fe)throw new w(`BackingDataset/${f.name} external dependency '${M}' does not exist for this Frank tenant`);let te=It(fe.status);if(te!=="synced")throw new w(`BackingDataset/${f.name} external dependency '${M}' has status=${te||"<missing>"}; it must already be synced before the dependent run can start`)}try{let O=await t.post(`/api/v1/backing-datasets/${f.id}/sync`,{});if(m=String(O.workflow_id??"").trim(),v=String(O.sync_run_id??"").trim(),!m||!v)throw new w(`BackingDataset/${f.name} sync response omitted sync_run_id/workflow_id; cannot correlate --wait.`)}catch(O){let M=lP(O);if(!M.matches)throw O;if(m=M.workflowId,v=M.syncRunId,P=!0,!m||!v)throw new w(`BackingDataset/${f.name} returned sync_already_running without sync_run_id/workflow_id.`);i(`  \u21B3 adopted already-running ontology sync_run_id=${v} workflow_id=${m}`)}if(!m||!v)throw new w(`BackingDataset/${f.name} did not provide exact ontology run identifiers.`);let b=await t.get(`/api/v1/backing-datasets/${f.id}`);if(b.id!==f.id)throw new w(`BackingDataset/${f.name} detail correlation mismatch: expected id=${f.id}, actual=${b.id}.`);let x=b.transform_id?d.get(b.transform_id):b.pipeline_id?p.get(b.pipeline_id):void 0;if(b.pipeline_id!=null&&s.has(b.pipeline_id)&&!x)throw new w(`BackingDataset/${f.name} is linked to applied Pipeline/${b.pipeline_id}, but transform_id=${b.transform_id??"<missing>"} has no exact TransformRun snapshot from this invocation.`);if(P&&!x)throw new w(`BackingDataset/${f.name} cannot adopt already-running ontology sync_run_id=${v} without an exact correlated TransformRun snapshot from this invocation.`);let C=await uP(t,f.id,v,m,r);if(z=It(C.status),z!=="synced"&&z!=="skipped")throw new w(`BackingDataset/${f.name} ontology workflow failed: ${z} workflow_id=${m} ontology_sync_run_id=${C.id} ${C.error_message??""}`);if(C.snapshot_id==null||C.rows_synced==null)throw new w(`BackingDataset/${f.name} ontology workflow completed without snapshot_id/rows_synced evidence (workflow_id=${m}, ontology_sync_run_id=${C.id}).`);let A=cm(C.snapshot_id,`BackingDataset/${f.name} (workflow_id=${m}, ontology_sync_run_id=${C.id})`);if(x&&A!==x.snapshotId)throw new w(`BackingDataset/${f.name} ontology snapshot mismatch: expected=${x.snapshotId} from transform_run_id=${x.transformRunId}, actual=${A} from ontology_sync_run_id=${C.id}.`);o.backing_datasets.push({name:f.name,id:f.id,workflow_id:m,ontology_sync_run_id:C.id,...x?{correlated_transform_id:x.transformId}:{},...x?{correlated_transform_run_id:x.transformRunId}:{},status:z,row_count:C.rows_synced,snapshot_id:A,bulk_jobs:Array.isArray(C.bulk_jobs)?C.bulk_jobs:[],...C.temporal_ui_url?{temporal_ui_url:C.temporal_ui_url}:{}}),f.backing_dataset_identity&&l.add(Le(f.backing_dataset_identity)),i(z==="skipped"?`  \u2713 ontology already converged; no replay workflow_id=${m} ontology_sync_run_id=${C.id} snapshot=${A} rows=0`:`  \u2713 ontology synced workflow_id=${m} ontology_sync_run_id=${C.id} snapshot=${A} rows=${C.rows_synced}`)}catch(b){let x=b.message;o.backing_datasets.push({name:f.name,id:f.id,...m?{workflow_id:m}:{},...v?{ontology_sync_run_id:v}:{},status:z,error:x}),i(`  \u2717 BackingDataset/${f.name}: ${x}`),n=!1}}return g()}var pP=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;async function fP(t,e,r){let i=e;if(!pP.test(e)){let u=await Rb(t,{sourceNameToId:new Map,pipelineNameToId:new Map},e);if(!u)throw new w(`Source '${e}' not found by name. Pass a source id or its metadata.name.`);i=u}let n=await t.get(`/api/v1/sources/${i}`,{include_streams:"true"}),o=n.streams??[];if(o.length===0)throw new w(`Source '${n.name}' has no streams to scaffold from. Configure a stream (and sync it to bronze) first.`);let s;if(r){if(s=o.find(l=>l.name===r),!s)throw new w(`Stream '${r}' not on source '${n.name}'. Streams: ${o.map(l=>l.name).join(", ")}`)}else{let l=o.filter(d=>d.is_enabled!==!1),u=l.length?l:o;if(u.length>1)throw new w(`Source '${n.name}' has ${u.length} streams \u2014 pass --stream <name> to choose one (${u.map(d=>d.name).join(", ")}).`);s=u[0]}if(!s.dataset_id)throw new w(`Stream '${s.name}' has no resolved bronze dataset_id; cannot read its physical schema.`);let a;try{a=await t.get(`/api/v1/datasets/${s.dataset_id}/physical-schema`)}catch(l){throw l instanceof F&&l.status===404?new w(`No physical schema for '${s.dataset_id}' yet \u2014 the bronze table (and its nested ROW columns) only exists after the source syncs. Run a sync (e.g. 'frankctl sources sync ${n.name}'), then scaffold.`):l}a.columns.length===0&&process.stderr.write(`warning: physical schema for '${s.dataset_id}' has 0 columns; emitting a 'SELECT *' skeleton
+wait: \u2717 one or more stages failed (see above).`),o);for(let f of e.filter(m=>m.kind==="Source")){i(`wait: syncing Source/${f.name} \u2192 bronze...`);let m,v;try{let P=await t.get(`/api/v1/sources/${f.id}`),z=Array.isArray(P.streams)?P.streams:[],b=new Map(z.map(V=>[String(V.name??"").trim(),V])),x=f.declared_stream_names?[...f.declared_stream_names]:z.filter(V=>V.is_enabled!==!1).map(V=>String(V.name??"").trim()).filter(Boolean);if(f.declared_stream_names){if(x.length===0)throw new w(`Source/${f.name} declares no enabled streams; --wait will not fall back to unmanaged streams.`);for(let V of x){let fe=b.get(V);if(!fe||fe.is_enabled===!1)throw new w(`Source/${f.name} declared enabled stream '${V}', but the applied Source does not expose it as enabled.`)}}P.status==="draft"&&await t.patch(`/api/v1/sources/${f.id}`,{status:"ready"});let D=await t.post(`/api/v1/sources/${f.id}/sync`,f.declared_stream_names?{stream_names:x}:{});if(m=String(D.workflow_id??"").trim(),!m)throw new w(`Source/${f.name} sync response omitted workflow_id; cannot correlate --wait.`);let C=m,A=await ye({fetch:()=>sP(t,C,f.id),isTerminal:V=>nP.has(It(V.status)),intervalMs:Math.min(5e3,r),timeoutMs:r});v=A.value;let O=It(A.value.status),M=A.timedOut?"timeout":O;if(A.timedOut||O!=="completed")o.sources.push({name:f.name,id:f.id,workflow_id:m,...A.value.sync_run_id?{sync_run_id:A.value.sync_run_id}:{},...A.value.workflow_run_id?{workflow_run_id:A.value.workflow_run_id}:{},status:M,...A.value.error_message?{error:A.value.error_message}:{},...A.value.temporal_ui_url?{temporal_ui_url:A.value.temporal_ui_url}:{}}),i(`  \u2717 Source/${f.name}: ${M||"error"} workflow_id=${m}`),n=!1;else{if(A.value.records_synced==null||A.value.total_streams==null||A.value.successful_streams==null||A.value.failed_streams==null)throw new w(`Source/${f.name} workflow_id=${m} completed without persisted stream/row metrics.`);if(A.value.total_streams!==D.streams_count)throw new w(`Source/${f.name} workflow_id=${m} persisted total_streams=${A.value.total_streams}, but the trigger declared streams_count=${D.streams_count}.`);if(x.length>0&&x.length!==D.streams_count)throw new w(`Source/${f.name} trigger declared streams_count=${D.streams_count}, but source details returned ${x.length} enabled stream(s).`);if(A.value.failed_streams!==0||A.value.successful_streams!==A.value.total_streams)throw new w(`Source/${f.name} workflow_id=${m} completed with ${A.value.successful_streams}/${A.value.total_streams} successful streams and ${A.value.failed_streams} failures.`);let V=await aP(t,f.id,x,D.streams_count);o.sources.push({name:f.name,id:f.id,workflow_id:m,sync_run_id:A.value.sync_run_id,...A.value.workflow_run_id?{workflow_run_id:A.value.workflow_run_id}:{},status:O,row_count:A.value.records_synced,persisted_metrics:{total_streams:A.value.total_streams,successful_streams:A.value.successful_streams,failed_streams:A.value.failed_streams,records_synced:A.value.records_synced,bytes_synced:A.value.bytes_synced,batches_completed:A.value.batches_completed,total_batches_estimated:A.value.total_batches_estimated,records_written_so_far:A.value.records_written_so_far,failed_batches:A.value.failed_batches},observed_bronze_refs:V,...A.value.temporal_ui_url?{temporal_ui_url:A.value.temporal_ui_url}:{}}),i(`  \u2713 bronze synced (${f.name}) workflow_id=${m} sync_run_id=${A.value.sync_run_id} rows=${A.value.records_synced} observed_main_refs=${V.map(fe=>`${fe.table_path}@${fe.snapshot_id}`).join(",")}`+(A.value.temporal_ui_url?` temporal=${A.value.temporal_ui_url}`:""))}}catch(P){let z=P.message;o.sources.push({name:f.name,id:f.id,...m?{workflow_id:m}:{},...v?.sync_run_id?{sync_run_id:v.sync_run_id}:{},...v?.workflow_run_id?{workflow_run_id:v.workflow_run_id}:{},status:"error",error:z}),i(`  \u2717 Source/${f.name}: ${z}`),n=!1}}if(!n)return i("wait: source stage failed; downstream transforms were not triggered."),g();for(let f of e.filter(m=>m.kind==="Pipeline")){i(`wait: materializing Pipeline/${f.name} \u2192 silver...`);try{let m=await t.get(`/api/v1/pipelines/${f.id}`,{include_version:"true"}),v=m.current_version?.steps??[],P=iP(f.name,v,m.current_version?.dag_spec),z=oP(P,m.current_version?.dag_spec);for(let b of P){let x,D,C="error";try{let A=await t.post(`/api/v1/transforms/${b.transformId}/materialize`,{wait_for_completion:!1,timeout:Math.min(3600,Math.max(10,Math.ceil(r/1e3)))});if(x=String(A.dagster_run_id??"").trim(),!A.success||!x)throw new w(`Transform '${b.stepName}' was not started: ${A.error_message??"materialize response omitted dagster_run_id"}.`);let O=await cP(t,x,r);if(D=O.id,C=It(O.status),C!=="completed")throw new w(`Transform '${b.stepName}' failed: ${C} dagster_run_id=${x} transform_run_id=${O.id} ${O.error_message??""}`);if(O.output_snapshot_id==null)throw new w(`Transform '${b.stepName}' completed without output_snapshot_id (dagster_run_id=${x}, transform_run_id=${O.id}).`);let M=cm(O.output_snapshot_id,`Transform '${b.stepName}' (dagster_run_id=${x}, transform_run_id=${O.id})`),V=O.rows_affected??O.output_row_count;if(V==null)throw new w(`Transform '${b.stepName}' completed without an observable row count (dagster_run_id=${x}, transform_run_id=${O.id}).`);o.transforms.push({pipeline_name:f.name,pipeline_id:f.id,step_name:b.stepName,transform_id:b.transformId,dagster_run_id:x,transform_run_id:O.id,status:C,row_count:V,snapshot_id:M}),d.set(b.transformId,{snapshotId:M,transformId:b.transformId,transformRunId:O.id,pipelineId:f.id}),i(`  \u2713 silver materialized step=${b.stepName} dagster_run_id=${x} transform_run_id=${O.id} snapshot=${M} rows=${V}`)}catch(A){let O=A.message;throw o.transforms.push({pipeline_name:f.name,pipeline_id:f.id,step_name:b.stepName,transform_id:b.transformId,...x?{dagster_run_id:x}:{},...D?{transform_run_id:D}:{},status:C,error:O}),A}}if(z){let b=d.get(z.transformId);b&&p.set(f.id,b)}}catch(m){i(`  \u2717 materialize ${f.name}: ${m.message}`),n=!1}}if(!n)return i("wait: transform stage failed; ontology sync was not triggered."),g();for(let f of a){i(`wait: syncing BackingDataset/${f.name} \u2192 ontology...`);let m,v,P=!1,z="error";try{for(let O of f.backing_dataset_dependencies??[]){let M=Le(O);if(c.get(M)){if(!l.has(M))throw new w(`BackingDataset/${f.name} dependency '${M}' did not reach terminal success in this invocation`);continue}u??=await tP(t);let fe=u.get(M);if(!fe)throw new w(`BackingDataset/${f.name} external dependency '${M}' does not exist for this Frank tenant`);let te=It(fe.status);if(te!=="synced")throw new w(`BackingDataset/${f.name} external dependency '${M}' has status=${te||"<missing>"}; it must already be synced before the dependent run can start`)}try{let O=await t.post(`/api/v1/backing-datasets/${f.id}/sync`,{});if(m=String(O.workflow_id??"").trim(),v=String(O.sync_run_id??"").trim(),!m||!v)throw new w(`BackingDataset/${f.name} sync response omitted sync_run_id/workflow_id; cannot correlate --wait.`)}catch(O){let M=lP(O);if(!M.matches)throw O;if(m=M.workflowId,v=M.syncRunId,P=!0,!m||!v)throw new w(`BackingDataset/${f.name} returned sync_already_running without sync_run_id/workflow_id.`);i(`  \u21B3 adopted already-running ontology sync_run_id=${v} workflow_id=${m}`)}if(!m||!v)throw new w(`BackingDataset/${f.name} did not provide exact ontology run identifiers.`);let b=await t.get(`/api/v1/backing-datasets/${f.id}`);if(b.id!==f.id)throw new w(`BackingDataset/${f.name} detail correlation mismatch: expected id=${f.id}, actual=${b.id}.`);let x=b.transform_id?d.get(b.transform_id):b.pipeline_id?p.get(b.pipeline_id):void 0;if(b.pipeline_id!=null&&s.has(b.pipeline_id)&&!x)throw new w(`BackingDataset/${f.name} is linked to applied Pipeline/${b.pipeline_id}, but transform_id=${b.transform_id??"<missing>"} has no exact TransformRun snapshot from this invocation.`);if(P&&!x)throw new w(`BackingDataset/${f.name} cannot adopt already-running ontology sync_run_id=${v} without an exact correlated TransformRun snapshot from this invocation.`);let C=await uP(t,f.id,v,m,r);if(z=It(C.status),z!=="synced"&&z!=="skipped")throw new w(`BackingDataset/${f.name} ontology workflow failed: ${z} workflow_id=${m} ontology_sync_run_id=${C.id} ${C.error_message??""}`);if(C.snapshot_id==null||C.rows_synced==null)throw new w(`BackingDataset/${f.name} ontology workflow completed without snapshot_id/rows_synced evidence (workflow_id=${m}, ontology_sync_run_id=${C.id}).`);let A=cm(C.snapshot_id_exact??C.snapshot_id,`BackingDataset/${f.name} (workflow_id=${m}, ontology_sync_run_id=${C.id})`);if(x&&A!==x.snapshotId)throw new w(`BackingDataset/${f.name} ontology snapshot mismatch: expected=${x.snapshotId} from transform_run_id=${x.transformRunId}, actual=${A} from ontology_sync_run_id=${C.id}.`);o.backing_datasets.push({name:f.name,id:f.id,workflow_id:m,ontology_sync_run_id:C.id,...x?{correlated_transform_id:x.transformId}:{},...x?{correlated_transform_run_id:x.transformRunId}:{},status:z,row_count:C.rows_synced,snapshot_id:A,bulk_jobs:Array.isArray(C.bulk_jobs)?C.bulk_jobs:[],...C.temporal_ui_url?{temporal_ui_url:C.temporal_ui_url}:{}}),f.backing_dataset_identity&&l.add(Le(f.backing_dataset_identity)),i(z==="skipped"?`  \u2713 ontology already converged; no replay workflow_id=${m} ontology_sync_run_id=${C.id} snapshot=${A} rows=0`:`  \u2713 ontology synced workflow_id=${m} ontology_sync_run_id=${C.id} snapshot=${A} rows=${C.rows_synced}`)}catch(b){let x=b.message;o.backing_datasets.push({name:f.name,id:f.id,...m?{workflow_id:m}:{},...v?{ontology_sync_run_id:v}:{},status:z,error:x}),i(`  \u2717 BackingDataset/${f.name}: ${x}`),n=!1}}return g()}var pP=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;async function fP(t,e,r){let i=e;if(!pP.test(e)){let u=await Rb(t,{sourceNameToId:new Map,pipelineNameToId:new Map},e);if(!u)throw new w(`Source '${e}' not found by name. Pass a source id or its metadata.name.`);i=u}let n=await t.get(`/api/v1/sources/${i}`,{include_streams:"true"}),o=n.streams??[];if(o.length===0)throw new w(`Source '${n.name}' has no streams to scaffold from. Configure a stream (and sync it to bronze) first.`);let s;if(r){if(s=o.find(l=>l.name===r),!s)throw new w(`Stream '${r}' not on source '${n.name}'. Streams: ${o.map(l=>l.name).join(", ")}`)}else{let l=o.filter(d=>d.is_enabled!==!1),u=l.length?l:o;if(u.length>1)throw new w(`Source '${n.name}' has ${u.length} streams \u2014 pass --stream <name> to choose one (${u.map(d=>d.name).join(", ")}).`);s=u[0]}if(!s.dataset_id)throw new w(`Stream '${s.name}' has no resolved bronze dataset_id; cannot read its physical schema.`);let a;try{a=await t.get(`/api/v1/datasets/${s.dataset_id}/physical-schema`)}catch(l){throw l instanceof F&&l.status===404?new w(`No physical schema for '${s.dataset_id}' yet \u2014 the bronze table (and its nested ROW columns) only exists after the source syncs. Run a sync (e.g. 'frankctl sources sync ${n.name}'), then scaffold.`):l}a.columns.length===0&&process.stderr.write(`warning: physical schema for '${s.dataset_id}' has 0 columns; emitting a 'SELECT *' skeleton
 `);let c=Nb(n,s,a);return Cb(c,n.name,s.name)}function Mb(t){let e=t.command("pipelines").description("Manage pipelines");e.command("list").description("List pipelines").option("--status <status>","filter by status").option("--search <q>","search string").option("--page <n>","page number",r=>parseInt(r,10)).option("--page-size <n>","page size",r=>parseInt(r,10)).action(async(r,i)=>{let n=i.optsWithGlobals(),o=qt(n),s=await Ke(()=>o.get("/api/v1/pipelines",{status:r.status,search:r.search,page:r.page,page_size:r.pageSize}));if(n.json)return T(s);Y(s.items,[{key:"id",header:"ID"},{key:"name",header:"NAME"},{key:"status",header:"STATUS"},{key:"active_version_id",header:"ACTIVE_VERSION"}])}),e.command("get <id>").description("Get a pipeline by id").option("--include-version","include active version payload").action(async(r,i,n)=>{let o=n.optsWithGlobals(),s=qt(o),a=await Ke(()=>s.get(`/api/v1/pipelines/${r}`,{include_version:i.includeVersion?"true":void 0}));if(o.json)return T(a);Q([["id",a.id],["name",a.name],["status",a.status],["active_version_id",a.active_version_id??""],["updated_at",a.updated_at??""]])}),e.command("delete <id>").description("Delete a pipeline (does not delete its silver tables)").option("--yes","skip confirmation").action(async(r,i,n)=>{let o=n.optsWithGlobals();i.yes||(process.stderr.write(`delete pipeline ${r}? re-run with --yes to confirm
 `),process.exit(2));let s=qt(o);await Ke(()=>s.del(`/api/v1/pipelines/${r}`)),o.json||process.stderr.write(`deleted ${r}
 `)}),e.command("apply").description("Apply a multi-doc YAML file (Source / Pipeline / BackingDataset) idempotently").requiredOption("-f, --file <path>","YAML file path").option("--allow-deprecate","Allow ensure_schema narrowing (passed through to BackingDataset POSTs)").option("--allow-recreate","On 409 immutable_diff (PR-C), DELETE the existing row and re-POST. Operational state (cursor, ledger, snapshot ids, run history) is LOST \u2014 explicit opt-in only. Default off; the apply path is idempotent without it.").option("--dry-run","Client-side parse + resolve. Prints the apply plan without talking to the server. Catches malformed YAML and obviously broken cross-refs; does NOT validate pattern_ids or detect immutable_diff. Use --dry-run-server for that.").option("--dry-run-server","Server-side preflight. Sends the multi-doc body to the dry-run endpoint which validates each doc against the catalog and DB without persisting. Catches missing pattern_ids, would-409 immutable diffs, and unresolved cross-refs. Does not contact the ontology unless --live-ontology is also passed.").option("--live-ontology","With --dry-run-server, add GET-only compatibility checks against the configured live ontology for BackingDatasets and render the returned endpoint/schema evidence. Invalid without --dry-run-server.").option("--wait","After applying, drive the chain end-to-end: sync Sources \u2192 bronze, materialize Pipeline steps \u2192 silver, and sync BackingDatasets \u2192 ontology, polling each to terminal. Exits non-zero if any stage fails.").option("--timeout <sec>","Per-stage timeout for --wait (default 300)",r=>parseInt(r,10),300).action(async(r,i)=>{let n=i.optsWithGlobals(),o=qt(n),s;try{s=VT(BT(r.file))}catch(u){throw u instanceof U&&(process.stderr.write(`${u.message}
@@ -350,7 +350,7 @@ rationale: ${l.rationale||"(none)"}
 `),T(l),process.exit(5));let u=EP(l);process.stdout.write(u);return}T(l)}),r.command("pattern-params").description("Suggest pattern parameters from intent").requiredOption("-f, --file <path>","spec file with {pattern_id, intent}").action(async(i,n)=>{let o=n.optsWithGlobals(),s=ot(o),a=await le(async()=>ee(i.file,t$)),c=Tt(await le(()=>s.post("/api/v1/ai/suggest-pattern-params",a)));T(c)}),e.command("publish-transform").description("Open a PR against frank-transforms with generated files").requiredOption("-f, --file <path>","spec file with {pattern_id, files}").action(async(i,n)=>{let o=n.optsWithGlobals(),s=ot(o),a=await le(async()=>ee(i.file,n$)),c=Tt(await le(()=>s.post("/api/v1/ai/publish-transform",a)));T(c)})}function NP(t){let e=J({profile:t.profile,apiUrl:t.apiUrl}),r={enabled:process.env.FRANK_DEV_MODE==="true",tenantId:process.env.FRANKCTL_TENANT_ID};return new oe({profile:e,getToken:()=>ie({profile:t.profile,apiUrl:t.apiUrl}),devMode:r})}async function g$(t){try{return await t()}catch(e){throw(e instanceof F||e instanceof w)&&(process.stderr.write(`${e.message}
 `),process.exit(e.exitCode)),e}}function h$(t){t.command("admin").description("Operational maintenance").command("reconcile-runs").description("Sync stuck transform runs with their backend and heal zombie last_run_outcome=RUNNING markers (transforms that can never be triggered because their cache says a run is in flight when none is)").option("--force","reconcile all active runs, not just stale ones",!1).option("--dry-run","list zombie transforms that would be healed; change nothing",!1).action(async(r,i)=>{let n=i.optsWithGlobals(),o=NP(n);if(r.dryRun){let a=await g$(()=>o.get("/api/v1/admin/zombie-transforms"));if(n.json)return T(a);if(!a.length){process.stderr.write(`no zombie transforms
 `);return}Y(a,[{key:"id",header:"ID"},{key:"current_outcome",header:"CURRENT"},{key:"would_heal_to",header:"WOULD_HEAL_TO"}]);return}let s=await g$(()=>o.post(`/api/v1/admin/reconcile-runs?force_all=${r.force?"true":"false"}`));if(n.json)return T(s);process.stderr.write(`${s.message}
-`)})}import y$ from"node:fs";import li from"node:path";import{fileURLToPath as CP}from"node:url";function DP(){let t=li.dirname(CP(import.meta.url)),e=[li.join(t,"skills","frank-cli","SKILL.md"),li.join(t,"..","skills","frank-cli","SKILL.md"),li.join(t,"..","..","..","skills","frank-cli","SKILL.md"),li.join(t,"..","..","skills","frank-cli","SKILL.md")];for(let r of e)if(y$.existsSync(r))return r;return null}async function jP(){let t=DP();if(!t)throw new w("SKILL.md not found in the installed package. Reinstall via `npm i -g @aiaiai-pt/frankctl` or run `npx -y @aiaiai-pt/frankctl@latest skill`.",1);process.stdout.write(y$.readFileSync(t,"utf-8"))}function v$(t){t.command("skill").description("Print the bundled agent skill reference (SKILL.md) to stdout").action(async()=>{await jP()})}var RP="0.5.0";function LP(){let t=new zm;return t.name("frankctl").description("Frank CLI \u2014 scriptable access to the Frank API").version(RP).option("-p, --profile <name>","config profile to use").option("--json","emit machine-readable JSON output").option("-v, --verbose","verbose logging").option("-q, --quiet","suppress non-error output").option("--no-color","disable ANSI colors").option("--api-url <url>","override the API base URL").showHelpAfterError(),Ib(t),Pb(t),Ab(t),Mb(t),Fb(t),s$(t),c$(t),l$(t),u$(t),d$(t),p$(t),m$(t),h$(t),v$(t),t}function Nj(t){return t.optsWithGlobals()}function Cj(t){return new Jn({verbose:t.verbose??!1,quiet:t.quiet??!1,color:t.color!==!1})}async function MP(t){let e=LP();e.exitOverride();try{return await e.parseAsync(t),0}catch(r){if(r instanceof w)return process.stderr.write(`error: ${r.message}
+`)})}import y$ from"node:fs";import li from"node:path";import{fileURLToPath as CP}from"node:url";function DP(){let t=li.dirname(CP(import.meta.url)),e=[li.join(t,"skills","frank-cli","SKILL.md"),li.join(t,"..","skills","frank-cli","SKILL.md"),li.join(t,"..","..","..","skills","frank-cli","SKILL.md"),li.join(t,"..","..","skills","frank-cli","SKILL.md")];for(let r of e)if(y$.existsSync(r))return r;return null}async function jP(){let t=DP();if(!t)throw new w("SKILL.md not found in the installed package. Reinstall via `npm i -g @aiaiai-pt/frankctl` or run `npx -y @aiaiai-pt/frankctl@latest skill`.",1);process.stdout.write(y$.readFileSync(t,"utf-8"))}function v$(t){t.command("skill").description("Print the bundled agent skill reference (SKILL.md) to stdout").action(async()=>{await jP()})}var RP="0.5.1";function LP(){let t=new zm;return t.name("frankctl").description("Frank CLI \u2014 scriptable access to the Frank API").version(RP).option("-p, --profile <name>","config profile to use").option("--json","emit machine-readable JSON output").option("-v, --verbose","verbose logging").option("-q, --quiet","suppress non-error output").option("--no-color","disable ANSI colors").option("--api-url <url>","override the API base URL").showHelpAfterError(),Ib(t),Pb(t),Ab(t),Mb(t),Fb(t),s$(t),c$(t),l$(t),u$(t),d$(t),p$(t),m$(t),h$(t),v$(t),t}function Nj(t){return t.optsWithGlobals()}function Cj(t){return new Jn({verbose:t.verbose??!1,quiet:t.quiet??!1,color:t.color!==!1})}async function MP(t){let e=LP();e.exitOverride();try{return await e.parseAsync(t),0}catch(r){if(r instanceof w)return process.stderr.write(`error: ${r.message}
 `),r.exitCode;let i=r;return i.code==="commander.helpDisplayed"||i.code==="commander.version"?0:i.code?.startsWith("commander.")?(i.message&&process.stderr.write(`${i.message}
 `),2):(process.stderr.write(`error: ${i.message??String(r)}
 `),1)}}function ZP(){try{let t=UP(process.argv[1]??""),e=new URL(import.meta.url).pathname;return t===e}catch{return!1}}ZP()&&MP(process.argv).then(t=>process.exit(t));export{LP as buildProgram,MP as main,Cj as makeLogger,Nj as resolveGlobalOptions};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiaiai-pt/frankctl",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Terminal-first client for the Frank Low-Code Pipeline platform",
   "homepage": "https://franksdocs.nomadriver.co",
   "repository": {

package/skills/frank-cli/SKILL.md

@@ -108,8 +108,9 @@ frankctl pipelines apply -f pipeline.yaml --wait --timeout 600
 Use `--json` with `--wait` for one evidence object correlated to the exact Source
 workflow/SyncRun, Dagster/TransformRun, and ontology SyncRun started by this invocation.
 Snapshot IDs are decimal strings so JavaScript cannot round them. `status: skipped`
-means the ontology was already converged and no rows were replayed; use `bd entities`
-for separate target entity readback.
+means the ontology was already converged and no rows were replayed. Ontology evidence
+prefers `snapshot_id_exact`; numeric-only evidence fails closed. Use `bd entities` for
+separate target entity readback.
 
 ### Preflight before you apply