@tiangong-lca/cli 0.1.0 → 0.1.2

package/dist/src/lib/dataset-save-draft-run.d.ts

@@ -0,0 +1,283 @@
+import * as tidasSdk from '@tiangong-lca/tidas-sdk';
+import { type DatasetCommandTable } from './dataset-command.js';
+import type { FetchLike } from './http.js';
+import { type SupabaseDataRuntime } from './supabase-client.js';
+import { type RemoteDatasetLookup, type RemoteDatasetReference, type RemoteDatasetTable } from './dataset-remote-verify.js';
+type JsonObject = Record<string, unknown>;
+export type DatasetSaveDraftType = 'auto' | 'contact' | 'source' | 'unitgroup' | 'flowproperty' | 'flow' | 'process';
+type ConcreteDatasetSaveDraftType = Exclude<DatasetSaveDraftType, 'auto'>;
+type DatasetTypeConfig = {
+    table: DatasetCommandTable;
+    rootKey: string;
+    informationKey: string;
+    schemaName: keyof typeof tidasSdk;
+    factoryName: keyof typeof tidasSdk;
+};
+type DatasetSaveDraftValidationIssue = {
+    path: string;
+    message: string;
+    code: string;
+};
+type DatasetSaveDraftValidationResult = {
+    ok: true;
+    validator: string;
+    issue_count: 0;
+    issues: [];
+} | {
+    ok: false;
+    validator: string;
+    issue_count: number;
+    issues: DatasetSaveDraftValidationIssue[];
+};
+declare function normalizeValidationIssue(issue: {
+    path?: Array<string | number>;
+    message?: string;
+    code?: string;
+}): DatasetSaveDraftValidationIssue;
+export type DatasetSaveDraftRowReport = {
+    index: number;
+    id: string | null;
+    version: string | null;
+    type: ConcreteDatasetSaveDraftType | null;
+    table: DatasetCommandTable | null;
+    status: 'prepared' | 'executed' | 'failed' | 'unknown' | 'blocked';
+    operation: 'would_sync' | 'insert' | 'save_draft' | 'skipped_invalid' | 'reference_only_type' | 'type_unknown' | 'identity_missing' | 'elementary_flow_insert_blocked' | 'remote_reference_unresolved' | 'recovered_exact_readback' | 'blocked_dependency' | null;
+    validation: DatasetSaveDraftValidationResult | null;
+    visible_row?: VisibleDatasetRow | null;
+    error?: {
+        message: string;
+        details?: unknown;
+    };
+    action_id?: string;
+    desired_sha256?: string;
+    attempt_consumed?: boolean;
+    replayed?: false;
+    readback?: 'desired_exact' | 'not_desired' | 'not_performed';
+};
+export type DatasetSaveDraftReport = {
+    schema_version: 1 | 2;
+    generated_at_utc: string;
+    input_path: string;
+    requested_type: DatasetSaveDraftType;
+    out_dir: string;
+    commit: boolean;
+    mode: 'dry_run' | 'commit';
+    status: 'completed' | 'completed_with_failures' | 'completed_with_unknowns';
+    counts: {
+        selected: number;
+        prepared: number;
+        executed: number;
+        failed: number;
+        unknown?: number;
+        blocked?: number;
+        attempts_consumed?: number;
+        by_table: Partial<Record<DatasetCommandTable, number>>;
+        operations: Record<string, number>;
+    };
+    files: {
+        selected_rows: string;
+        progress_jsonl: string;
+        failures_jsonl: string;
+        summary_json: string;
+        execution_ledger?: string;
+    };
+    rows: DatasetSaveDraftRowReport[];
+    execution_contract?: {
+        path: string;
+        sha256: string;
+        execution_id: string;
+        target_mode: 'owner_draft';
+        max_parallel: number;
+        serial_prefix_actions: number;
+        parallel_suffix_actions: number;
+    };
+};
+export type RunDatasetSaveDraftOptions = {
+    inputPath: string;
+    type?: string | null;
+    outDir?: string | null;
+    commit?: boolean | null;
+    rawInput?: unknown;
+    env?: NodeJS.ProcessEnv;
+    fetchImpl?: FetchLike;
+    timeoutMs?: number;
+    now?: Date;
+    /**
+     * Explicit opt-in to write account-local (My Data, state_code=0) Unit Group and
+     * Flow Property support rows that are otherwise reference-only. Default false keeps
+     * the CLI safe-by-default for interactive operators.
+     */
+    allowReferenceOnlySupport?: boolean | null;
+    executionContractPath?: string | null;
+    maxParallel?: number | null;
+};
+type DatasetSaveDraftExecutionAction = {
+    action_id: string;
+    desired_sha256: string;
+    expected_operation: 'insert' | 'save_draft';
+    table: DatasetCommandTable;
+    id: string;
+    version: string;
+    before_sha256: string | null;
+    dependency_action_ids: string[];
+};
+type DatasetSaveDraftExecutionContract = {
+    schema_version: 'dataset-save-draft-execution-contract.v1';
+    execution_id: string;
+    project_ref: string;
+    target_mode: 'owner_draft';
+    owner: {
+        user_id: string;
+        email: string;
+        state_code: 0;
+    };
+    actions: DatasetSaveDraftExecutionAction[];
+};
+type DatasetSaveDraftLedgerEvent = {
+    schema_version: 'dataset-save-draft-execution-event.v1';
+    sequence: number;
+    contract_sha256: string;
+    action_id: string;
+    desired_sha256: string;
+    action_binding_sha256: string;
+    event_type: 'attempt_emitted' | 'outcome';
+    operation: 'insert' | 'save_draft';
+    outcome: 'executed' | 'unknown' | null;
+    recovered: boolean;
+    recorded_at_utc: string;
+    previous_event_sha256: string | null;
+    event_sha256: string;
+};
+type ExecutionLedgerState = {
+    events: Map<string, DatasetSaveDraftLedgerEvent[]>;
+    attempts: Map<string, DatasetSaveDraftLedgerEvent>;
+    outcomes: Map<string, DatasetSaveDraftLedgerEvent>;
+};
+type PreparedDatasetRow = {
+    index: number;
+    row: JsonObject;
+    payload: JsonObject;
+    type: ConcreteDatasetSaveDraftType | null;
+    config: DatasetTypeConfig | null;
+    id: string | null;
+    version: string | null;
+    validation: DatasetSaveDraftValidationResult | null;
+};
+type VisibleDatasetRow = {
+    id: string;
+    version: string;
+    user_id: string | null;
+    state_code: number | null;
+};
+type ExecutionDatasetRow = VisibleDatasetRow & {
+    json_ordered: JsonObject | null;
+};
+type MissingFlowRemoteReference = {
+    table: RemoteDatasetTable | null;
+    id: string | null;
+    version: string | null;
+    path: string;
+    short_description: string | null;
+    status: 'missing_dataset' | 'missing_version' | 'unsupported_type' | 'version_missing' | 'version_outdated';
+    latest_version: string | null;
+};
+declare function parseExecutionContract(value: unknown): DatasetSaveDraftExecutionContract;
+declare function bindExecutionContractRows(contract: DatasetSaveDraftExecutionContract, rows: PreparedDatasetRow[]): void;
+declare function executionActionBindingSha256(action: DatasetSaveDraftExecutionAction): string;
+declare function executionLedgerRoot(env: NodeJS.ProcessEnv, contract: DatasetSaveDraftExecutionContract): string;
+declare function executionLedgerPath(ledgerRoot: string, action: DatasetSaveDraftExecutionAction): string;
+declare function parseLedgerEvent(value: unknown, index: number): DatasetSaveDraftLedgerEvent;
+declare function loadExecutionLedger(ledgerRoot: string, contract: DatasetSaveDraftExecutionContract): ExecutionLedgerState;
+declare function projectRefFromApiBaseUrl(apiBaseUrl: string): string;
+declare function decodeExecutionActor(accessToken: string): {
+    user_id: string;
+    email: string;
+};
+declare function serializeError(error: unknown): {
+    message: string;
+    details?: unknown;
+};
+declare function normalizeType(value: string | null | undefined, allowReferenceOnlySupport?: boolean): DatasetSaveDraftType;
+declare function unwrapPayload(row: JsonObject): JsonObject;
+declare function detectType(payload: JsonObject): ConcreteDatasetSaveDraftType | null;
+declare function validatePayload(payload: JsonObject, type: ConcreteDatasetSaveDraftType, config: DatasetTypeConfig): DatasetSaveDraftValidationResult;
+declare function extractIdentity(payload: JsonObject, row: JsonObject, config: DatasetTypeConfig): {
+    id: string | null;
+    version: string | null;
+};
+declare function flowType(payload: JsonObject): string | null;
+declare function isElementaryFlowPayload(payload: JsonObject): boolean;
+declare function compareVersions(left: string | null, right: string | null): number;
+declare function supportLookupKey(reference: {
+    table: RemoteDatasetTable;
+    id: string;
+    version: string | null;
+}): string;
+declare function isLookupableRemoteReference(reference: RemoteDatasetReference): reference is RemoteDatasetReference & {
+    table: RemoteDatasetTable;
+    id: string;
+};
+declare function uniqueFlowRemoteReferences(payload: JsonObject): RemoteDatasetReference[];
+declare function remoteReferenceFallbackKey(reference: Pick<RemoteDatasetReference, 'path' | 'type'>): string;
+declare function missingFlowRemoteReferences(options: {
+    runtime: SupabaseDataRuntime;
+    fetchImpl: FetchLike;
+    timeoutMs: number;
+    cache: Map<string, Promise<RemoteDatasetLookup>>;
+    payload: JsonObject;
+}): Promise<MissingFlowRemoteReference[]>;
+declare function prepareRows(inputPath: string, rawInput: unknown | undefined, requestedType: DatasetSaveDraftType): PreparedDatasetRow[];
+declare function buildFiles(outDir: string): DatasetSaveDraftReport['files'];
+declare function defaultOutDir(inputPath: string, commit: boolean, now: Date): string;
+declare function operationCount(rows: DatasetSaveDraftRowReport[]): Record<string, number>;
+declare function byTable(rows: PreparedDatasetRow[]): Partial<Record<DatasetCommandTable, number>>;
+declare function selectedRow(row: PreparedDatasetRow): JsonObject;
+declare function buildPreparedFailure(row: PreparedDatasetRow, allowReferenceOnlySupport?: boolean): DatasetSaveDraftRowReport | null;
+declare function buildVisibleRowsUrl(restBaseUrl: string, table: DatasetCommandTable, id: string, version: string): string;
+declare function parseVisibleRows(payload: unknown, url: string): VisibleDatasetRow[];
+declare function parseExecutionRows(payload: unknown, url: string): ExecutionDatasetRow[];
+declare function exactDesiredReadback(options: {
+    rows: ExecutionDatasetRow[];
+    action: DatasetSaveDraftExecutionAction;
+    contract: DatasetSaveDraftExecutionContract;
+}): boolean;
+export declare function runDatasetSaveDraft(options: RunDatasetSaveDraftOptions): Promise<DatasetSaveDraftReport>;
+export declare const __testInternals: {
+    DATASET_CONFIGS: Record<ConcreteDatasetSaveDraftType, DatasetTypeConfig>;
+    buildFiles: typeof buildFiles;
+    buildPreparedFailure: typeof buildPreparedFailure;
+    buildVisibleRowsUrl: typeof buildVisibleRowsUrl;
+    byTable: typeof byTable;
+    compareVersions: typeof compareVersions;
+    defaultOutDir: typeof defaultOutDir;
+    detectType: typeof detectType;
+    decodeExecutionActor: typeof decodeExecutionActor;
+    bindExecutionContractRows: typeof bindExecutionContractRows;
+    executionLedgerRoot: typeof executionLedgerRoot;
+    executionLedgerPath: typeof executionLedgerPath;
+    executionActionBindingSha256: typeof executionActionBindingSha256;
+    loadExecutionLedger: typeof loadExecutionLedger;
+    exactDesiredReadback: typeof exactDesiredReadback;
+    extractIdentity: typeof extractIdentity;
+    flowType: typeof flowType;
+    isElementaryFlowPayload: typeof isElementaryFlowPayload;
+    isLookupableRemoteReference: typeof isLookupableRemoteReference;
+    missingFlowRemoteReferences: typeof missingFlowRemoteReferences;
+    normalizeValidationIssue: typeof normalizeValidationIssue;
+    normalizeType: typeof normalizeType;
+    operationCount: typeof operationCount;
+    parseVisibleRows: typeof parseVisibleRows;
+    parseExecutionContract: typeof parseExecutionContract;
+    parseExecutionRows: typeof parseExecutionRows;
+    parseLedgerEvent: typeof parseLedgerEvent;
+    projectRefFromApiBaseUrl: typeof projectRefFromApiBaseUrl;
+    prepareRows: typeof prepareRows;
+    remoteReferenceFallbackKey: typeof remoteReferenceFallbackKey;
+    selectedRow: typeof selectedRow;
+    serializeError: typeof serializeError;
+    supportLookupKey: typeof supportLookupKey;
+    unwrapPayload: typeof unwrapPayload;
+    uniqueFlowRemoteReferences: typeof uniqueFlowRemoteReferences;
+    validatePayload: typeof validatePayload;
+};
+export {};

package/dist/src/lib/dataset-save-draft-run.js

@@ -3,6 +3,7 @@
 import { closeSync, chmodSync, fsyncSync, mkdirSync, openSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import * as tidasSdk from '@tiangong-lca/tidas-sdk';
+import { createBatchContract, runBoundedBatch } from '../batch.js';
 import { writeJsonArtifact, writeJsonLinesArtifact } from './artifacts.js';
 import { collectImportContentIssues } from './dataset-validate.js';
 import { createDatasetRecord, saveDraftDatasetRecord, } from './dataset-command.js';
@@ -937,6 +938,18 @@ function assertParallelSuffixTargetsAreUnique(contract, serialPrefixLength) {
         targets.add(target);
     }
 }
+function executionBatchActionContent(action) {
+    return {
+        action_id: action.action_id,
+        desired_sha256: action.desired_sha256,
+        expected_operation: action.expected_operation,
+        table: action.table,
+        id: action.id,
+        version: action.version,
+        before_sha256: action.before_sha256,
+        dependency_action_ids: [...action.dependency_action_ids],
+    };
+}
 async function renewExecutionOwnerToken(options) {
     const accessToken = await options.runtime.getAccessToken();
     const actor = decodeExecutionActor(accessToken);
@@ -1169,27 +1182,47 @@ async function runExecutionContractBatch(options) {
     for (let index = 0; index < serialPrefixLength; index += 1) {
         await executeAction(index);
     }
-    let nextParallelIndex = serialPrefixLength;
-    let fatalWorkerError = null;
-    const runParallelWorker = async () => {
-        while (fatalWorkerError === null) {
-            const index = nextParallelIndex;
-            nextParallelIndex += 1;
-            if (index >= options.contract.actions.length) {
-                return;
-            }
-            try {
-                await executeAction(index);
-            }
-            catch (error) {
-                fatalWorkerError ??= error;
-            }
-        }
-    };
-    const parallelWorkerCount = Math.min(options.maxParallel, options.contract.actions.length - serialPrefixLength);
-    await Promise.all(Array.from({ length: parallelWorkerCount }, () => runParallelWorker()));
-    if (fatalWorkerError !== null) {
-        throw fatalWorkerError;
+    const parallelIndexes = options.contract.actions
+        .slice(serialPrefixLength)
+        .map((_action, suffixIndex) => serialPrefixLength + suffixIndex);
+    const parallelBatch = await runBoundedBatch({
+        contract: createBatchContract({
+            identity: {
+                schema: 'dataset-save-draft.parallel-suffix.v1',
+                execution_id: options.contract.execution_id,
+                contract_sha256: contractSha256,
+            },
+            content: parallelIndexes.map((index) => executionBatchActionContent(options.contract.actions[index])),
+            policy: {
+                max_parallel: options.maxParallel,
+                serial_prefix_actions: serialPrefixLength,
+                mutation_retry: 'none',
+                fatal_stop: true,
+            },
+        }),
+        items: parallelIndexes,
+        getItemIdentity: (index) => options.contract.actions[index].action_id,
+        projectItemContent: (index) => executionBatchActionContent(options.contract.actions[index]),
+        projectItemPolicy: (index) => {
+            const action = options.contract.actions[index];
+            return {
+                contract_sha256: contractSha256,
+                target_mode: 'owner_draft',
+                expected_operation: action.expected_operation,
+            };
+        },
+        getExclusiveKey: ({ item: index }) => {
+            const action = options.contract.actions[index];
+            return JSON.stringify([action.table, action.id, action.version]);
+        },
+        mode: 'mutation',
+        maxConcurrency: options.maxParallel,
+        execute: async ({ item: index }) => executeAction(index),
+        shouldStop: ({ last_result: lastResult }) => lastResult.status === 'failed',
+    });
+    const fatalWorkerResult = parallelBatch.results_completion_order.find((result) => result.status === 'failed');
+    if (fatalWorkerResult?.status === 'failed') {
+        throw fatalWorkerResult.error;
     }
     const completedReports = reports;
     const failures = completedReports.filter((row) => ['failed', 'unknown', 'blocked'].includes(row.status));