@tiangong-lca/cli 0.0.11 → 0.0.13

package/dist/src/lib/dataset-curation-queue.js

@@ -32,6 +32,8 @@ export async function runDatasetCurationQueueBuild(options) {
     const flowTasks = flowRows.map((row) => buildTask(outDir, row, []));
     const processTasks = processRows.map((row) => {
         const refs = extractProcessFlowRefs(row.payload);
+        const deferredRefs = extractDeferredProcessFlowRefs(row.payload);
+        const deferredRefKeys = deferredProcessFlowRefKeys(deferredRefs);
         const dependsOn = new Set();
         const missingRefs = [];
         for (const ref of refs) {
@@ -39,7 +41,7 @@ export async function runDatasetCurationQueueBuild(options) {
             if (taskId) {
                 dependsOn.add(taskId);
             }
-            else if (!externalFlowIds.has(ref.id)) {
+            else if (!externalFlowIds.has(ref.id) && !deferredRefKeys.has(flowRefKey(ref))) {
                 missingRefs.push(ref);
             }
         }
@@ -68,6 +70,7 @@ export async function runDatasetCurationQueueBuild(options) {
             row,
             task,
             flowRefs: row.entityType === 'process' ? extractProcessFlowRefs(row.payload) : [],
+            deferredFlowRefs: row.entityType === 'process' ? extractDeferredProcessFlowRefs(row.payload) : [],
             externalFlowIds,
             flowRowsById,
         });
@@ -134,6 +137,250 @@ export async function runDatasetCurationQueueBuild(options) {
     writeText(blockersPath, jsonLines(blockers));
     return report;
 }
+export async function runDatasetCurationQueueNext(options) {
+    const queue = readQueueRuntime(options.queueDir);
+    const scope = normalizeQueueScope(options);
+    const taskStates = buildTaskStates(queue.tasks).filter((state) => taskMatchesScope(state, scope));
+    const scopedTasks = queue.tasks.filter((task) => taskMatchesScope(task, scope));
+    const stateByTaskId = new Map(taskStates.map((state) => [state.task_id, state]));
+    const runnable = scopedTasks.filter((task) => stateByTaskId.get(task.task_id)?.status === 'pending');
+    const counts = countTaskStates(taskStates);
+    const status = queue.blockers.length > 0 ||
+        (taskStates.length > 0 && runnable.length === 0 && counts.complete < taskStates.length)
+        ? 'blocked'
+        : counts.complete === taskStates.length
+            ? 'complete'
+            : 'ready';
+    const nextTask = status === 'ready' && runnable[0] ? withQueueAction(runnable[0]) : null;
+    return {
+        schema_version: 1,
+        generated_at_utc: new Date().toISOString(),
+        status,
+        queue_dir: queue.queueDir,
+        scope,
+        counts: {
+            ...counts,
+            runnable: runnable.length,
+        },
+        next_task: nextTask,
+        task_states: taskStates,
+        blockers: queue.blockers,
+    };
+}
+export async function runDatasetCurationQueueVerify(options) {
+    const queue = readQueueRuntime(options.queueDir);
+    const scope = normalizeQueueScope(options);
+    const taskStates = buildTaskStates(queue.tasks).filter((state) => taskMatchesScope(state, scope));
+    const counts = countTaskStates(taskStates);
+    return {
+        schema_version: 1,
+        generated_at_utc: new Date().toISOString(),
+        status: queue.blockers.length === 0 && counts.complete === taskStates.length ? 'passed' : 'blocked',
+        queue_dir: queue.queueDir,
+        scope,
+        counts,
+        task_states: taskStates,
+        blockers: queue.blockers,
+    };
+}
+function readQueueRuntime(queueDirInput) {
+    const queueDir = requireExistingPath(queueDirInput, '--queue-dir');
+    const outputsDir = path.join(queueDir, 'outputs');
+    const tasksPath = path.join(outputsDir, 'curation-queue-tasks.jsonl');
+    const blockersPath = path.join(outputsDir, 'curation-queue-blockers.jsonl');
+    if (!existsSync(tasksPath)) {
+        throw new CliError(`curation queue tasks file does not exist: ${tasksPath}`, {
+            code: 'CURATION_QUEUE_TASKS_NOT_FOUND',
+            exitCode: 2,
+        });
+    }
+    if (!existsSync(blockersPath)) {
+        throw new CliError(`curation queue blockers file does not exist: ${blockersPath}`, {
+            code: 'CURATION_QUEUE_BLOCKERS_NOT_FOUND',
+            exitCode: 2,
+        });
+    }
+    return {
+        queueDir: path.resolve(queueDir),
+        tasks: readJsonlFile(tasksPath).map((value, index) => parseQueueTask(value, tasksPath, index)),
+        blockers: readJsonlFile(blockersPath).map((value, index) => parseQueueBlocker(value, blockersPath, index)),
+    };
+}
+function normalizeQueueScope(options) {
+    return {
+        entity_type: options.entityType ?? null,
+        task_id: options.taskId?.trim() || null,
+    };
+}
+function buildTaskStates(tasks) {
+    const completeTaskIds = new Set(tasks
+        .filter((task) => isCompleteCheckpointStatus(readCheckpointStatus(task.checkpoint_file)))
+        .map((task) => task.task_id));
+    return tasks.map((task) => {
+        const checkpointStatus = readCheckpointStatus(task.checkpoint_file);
+        const incompleteDependencies = task.depends_on.filter((taskId) => !completeTaskIds.has(taskId));
+        const status = taskRuntimeStatus(checkpointStatus, incompleteDependencies);
+        return {
+            task_id: task.task_id,
+            entity_type: task.entity_type,
+            entity_id: task.entity_id,
+            version: task.version,
+            status,
+            checkpoint_status: checkpointStatus,
+            checkpoint_file: task.checkpoint_file,
+            depends_on: task.depends_on,
+            incomplete_dependencies: incompleteDependencies,
+            reason: status === 'waiting_dependencies'
+                ? 'dependency checkpoints are not complete'
+                : status === 'blocked'
+                    ? 'checkpoint status is blocked or failed'
+                    : null,
+        };
+    });
+}
+function taskRuntimeStatus(checkpointStatus, incompleteDependencies) {
+    if (isCompleteCheckpointStatus(checkpointStatus)) {
+        return 'complete';
+    }
+    if (checkpointStatus === 'blocked' || checkpointStatus === 'failed') {
+        return 'blocked';
+    }
+    if (incompleteDependencies.length > 0) {
+        return 'waiting_dependencies';
+    }
+    return 'pending';
+}
+function isCompleteCheckpointStatus(status) {
+    return (status === 'passed' || status === 'complete' || status === 'completed' || status === 'waived');
+}
+function readCheckpointStatus(checkpointFile) {
+    const resolvedPath = path.resolve(checkpointFile);
+    if (!existsSync(resolvedPath)) {
+        return null;
+    }
+    try {
+        const value = JSON.parse(readFileSync(resolvedPath, 'utf8'));
+        if (!isRecord(value)) {
+            return 'blocked';
+        }
+        const status = firstNonEmpty(value.status, value.state);
+        return status ? status.toLowerCase() : 'blocked';
+    }
+    catch {
+        return 'blocked';
+    }
+}
+function taskMatchesScope(task, scope) {
+    if (scope.entity_type && task.entity_type !== scope.entity_type) {
+        return false;
+    }
+    if (scope.task_id && task.task_id !== scope.task_id) {
+        return false;
+    }
+    return true;
+}
+function countTaskStates(taskStates) {
+    return {
+        total: taskStates.length,
+        complete: taskStates.filter((state) => state.status === 'complete').length,
+        pending: taskStates.filter((state) => state.status === 'pending').length,
+        waiting_dependencies: taskStates.filter((state) => state.status === 'waiting_dependencies')
+            .length,
+        blocked: taskStates.filter((state) => state.status === 'blocked').length,
+    };
+}
+function withQueueAction(task) {
+    return {
+        ...task,
+        action: {
+            kind: 'entity_curation_task',
+            input_artifact: task.input_rows_file,
+            output_artifacts: [task.checkpoint_file],
+        },
+    };
+}
+function readJsonlFile(inputPath) {
+    const text = readFileSync(inputPath, 'utf8').trim();
+    if (!text) {
+        return [];
+    }
+    return text.split(/\r?\n/u).map((line, index) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            throw new CliError(`Invalid JSONL in ${inputPath} at line ${index + 1}: ${String(error)}`, {
+                code: 'CURATION_QUEUE_JSONL_INVALID',
+                exitCode: 2,
+            });
+        }
+    });
+}
+function parseQueueTask(value, inputPath, index) {
+    if (!isRecord(value)) {
+        throw new CliError(`Invalid queue task in ${inputPath} at line ${index + 1}.`, {
+            code: 'CURATION_QUEUE_TASK_INVALID',
+            exitCode: 2,
+        });
+    }
+    const entityType = value.entity_type;
+    if (entityType !== 'support' && entityType !== 'flow' && entityType !== 'process') {
+        throw new CliError(`Invalid queue task entity_type in ${inputPath} at line ${index + 1}.`, {
+            code: 'CURATION_QUEUE_TASK_INVALID',
+            exitCode: 2,
+        });
+    }
+    return {
+        schema_version: 1,
+        entity_type: entityType,
+        task_id: requireString(value.task_id, inputPath, index, 'task_id'),
+        entity_id: requireString(value.entity_id, inputPath, index, 'entity_id'),
+        version: requireString(value.version, inputPath, index, 'version'),
+        lock_key: requireString(value.lock_key, inputPath, index, 'lock_key'),
+        depends_on: Array.isArray(value.depends_on)
+            ? value.depends_on.filter((item) => typeof item === 'string')
+            : [],
+        input_rows_file: requireString(value.input_rows_file, inputPath, index, 'input_rows_file'),
+        work_dir: requireString(value.work_dir, inputPath, index, 'work_dir'),
+        checkpoint_file: requireString(value.checkpoint_file, inputPath, index, 'checkpoint_file'),
+        run_plan_file: requireString(value.run_plan_file, inputPath, index, 'run_plan_file'),
+        closure_file: requireString(value.closure_file, inputPath, index, 'closure_file'),
+    };
+}
+function parseQueueBlocker(value, inputPath, index) {
+    if (!isRecord(value)) {
+        throw new CliError(`Invalid queue blocker in ${inputPath} at line ${index + 1}.`, {
+            code: 'CURATION_QUEUE_BLOCKER_INVALID',
+            exitCode: 2,
+        });
+    }
+    const entityType = value.entity_type;
+    if (entityType !== 'support' && entityType !== 'flow' && entityType !== 'process') {
+        throw new CliError(`Invalid queue blocker entity_type in ${inputPath} at line ${index + 1}.`, {
+            code: 'CURATION_QUEUE_BLOCKER_INVALID',
+            exitCode: 2,
+        });
+    }
+    return {
+        schema_version: 1,
+        code: requireString(value.code, inputPath, index, 'code'),
+        severity: 'blocker',
+        entity_type: entityType,
+        entity_id: typeof value.entity_id === 'string' ? value.entity_id : null,
+        version: typeof value.version === 'string' ? value.version : null,
+        message: requireString(value.message, inputPath, index, 'message'),
+        details: value.details,
+    };
+}
+function requireString(value, inputPath, index, field) {
+    if (typeof value === 'string' && value.length > 0) {
+        return value;
+    }
+    throw new CliError(`Invalid or missing ${field} in ${inputPath} at line ${index + 1}.`, {
+        code: 'CURATION_QUEUE_TASK_INVALID',
+        exitCode: 2,
+    });
+}
 function readQueueRows(inputPath, entityType) {
     const rows = materializeDatasetRows(inputPath);
     return rows.map((row) => queueRowFromDatasetRow(inputPath, entityType, row));
@@ -222,9 +469,11 @@ function buildDependencyClosure(options) {
         return {
             local_tasks: [],
             external_refs: [],
+            deferred_refs: [],
             unresolved_refs: [],
         };
     }
+    const deferredRefKeys = deferredProcessFlowRefKeys(options.deferredFlowRefs);
     return {
         local_tasks: options.flowRefs
             .map((ref) => ({ ref, row: options.flowRowsById.get(ref.id) }))
@@ -244,8 +493,25 @@ function buildDependencyClosure(options) {
             version: ref.version,
             ref_path: ref.path,
         })),
+        deferred_refs: options.flowRefs
+            .filter((ref) => !options.flowRowsById.has(ref.id) &&
+            !options.externalFlowIds.has(ref.id) &&
+            deferredRefKeys.has(flowRefKey(ref)))
+            .map((ref) => {
+            const deferredRef = deferredRefKeys.get(flowRefKey(ref));
+            return {
+                entity_type: 'flow',
+                entity_id: ref.id,
+                version: ref.version,
+                ref_path: ref.path,
+                action_item_code: deferredRef.actionItemCode,
+                reason: deferredRef.reason ?? null,
+            };
+        }),
         unresolved_refs: options.flowRefs
-            .filter((ref) => !options.flowRowsById.has(ref.id) && !options.externalFlowIds.has(ref.id))
+            .filter((ref) => !options.flowRowsById.has(ref.id) &&
+            !options.externalFlowIds.has(ref.id) &&
+            !deferredRefKeys.has(flowRefKey(ref)))
             .map((ref) => ({
             entity_type: 'flow',
             entity_id: ref.id,
@@ -289,12 +555,76 @@ function buildRunPlan(task) {
         },
     };
 }
+function extractDeferredProcessFlowRefs(payload) {
+    const refs = new Map();
+    scanForDeferredProcessFlowRefs(payload, refs);
+    return [...refs.values()].sort((a, b) => `${a.id}@${a.version ?? ''}@${a.path}`.localeCompare(`${b.id}@${b.version ?? ''}@${b.path}`));
+}
+function scanForDeferredProcessFlowRefs(value, refs) {
+    if (Array.isArray(value)) {
+        value.forEach((item) => scanForDeferredProcessFlowRefs(item, refs));
+        return;
+    }
+    if (!isRecord(value)) {
+        return;
+    }
+    const traces = asList(value['tiangongfoundry:unresolvedTrace']);
+    for (const trace of traces) {
+        if (!isRecord(trace)) {
+            continue;
+        }
+        const actionItemCode = firstNonEmpty(trace.action_item_code, trace.actionItemCode);
+        if (actionItemCode !== 'elementary_flow_identity_manual_review') {
+            continue;
+        }
+        const id = firstNonEmpty(trace.reference_id, trace.referenceId, trace.ref_object_id);
+        const pathValue = firstNonEmpty(trace.blocked_path, trace.blockedPath, trace.path);
+        if (!id || !pathValue) {
+            continue;
+        }
+        const version = firstNonEmpty(trace.reference_version, trace.referenceVersion, trace.version);
+        const ref = {
+            id,
+            version,
+            path: normalizeReferencePath(pathValue),
+            actionItemCode,
+            reason: firstNonEmpty(trace.reason),
+        };
+        refs.set(flowRefKey(ref), ref);
+    }
+    for (const nested of Object.values(value)) {
+        scanForDeferredProcessFlowRefs(nested, refs);
+    }
+}
+function deferredProcessFlowRefKeys(refs) {
+    const keys = new Map();
+    for (const ref of refs) {
+        keys.set(flowRefKey(ref), ref);
+    }
+    return keys;
+}
+function flowRefKey(ref) {
+    return `${ref.id}@${ref.version ?? ''}@${normalizeReferencePath(ref.path)}`;
+}
+function normalizeReferencePath(value) {
+    return value
+        .replace(/^\/+/u, '')
+        .replace(/\/+/gu, '.')
+        .replace(/^\.+|\.+$/gu, '');
+}
+function isFoundryTracePathParts(pathParts) {
+    return (pathParts.includes('common:other') &&
+        pathParts.some((part) => part.startsWith('tiangongfoundry:') && part.toLowerCase().includes('trace')));
+}
 function extractProcessFlowRefs(payload) {
     const refs = new Map();
     scanForFlowRefs(payload, [], refs);
     return [...refs.values()].sort((a, b) => `${a.id}@${a.version ?? ''}`.localeCompare(`${b.id}@${b.version ?? ''}`));
 }
 function scanForFlowRefs(value, pathParts, refs) {
+    if (isFoundryTracePathParts(pathParts)) {
+        return;
+    }
     if (Array.isArray(value)) {
         value.forEach((item, index) => scanForFlowRefs(item, [...pathParts, String(index)], refs));
         return;
@@ -340,6 +670,12 @@ function readExternalFlowRefs(inputPath) {
         };
     });
 }
+function asList(value) {
+    if (value === undefined || value === null) {
+        return [];
+    }
+    return Array.isArray(value) ? value : [value];
+}
 function normalizeProcessLimit(value) {
     if (value === undefined) {
         return null;
@@ -406,6 +742,27 @@ function jsonLines(rows) {
     return rows.map((row) => JSON.stringify(row)).join('\n') + (rows.length > 0 ? '\n' : '');
 }
 export const __testInternals = {
+    buildTaskStates,
+    countTaskStates,
+    entityDirName,
+    entityDirPlural,
+    extractDeferredProcessFlowRefs,
     extractProcessFlowRefs,
+    jsonLines,
+    normalizeReferencePath,
+    normalizeProcessLimit,
+    normalizeQueueScope,
+    parseQueueBlocker,
+    parseQueueTask,
+    readCheckpointStatus,
+    readJsonlFile,
+    readQueueRuntime,
+    requireExistingPath,
+    requirePath,
+    sanitizePathToken,
+    taskIdFor,
+    taskMatchesScope,
+    taskRuntimeStatus,
+    withQueueAction,
 };
 //# sourceMappingURL=dataset-curation-queue.js.map