@dtlabs/ai-toolkit 0.12.0 → 0.14.0

package/lib/execution-ledger.js

@@ -33,7 +33,8 @@ function _readLedger(ledgerPath) {
     }
     throw parseErr;
   }
-  return { entries: Array.isArray(parsed) ? parsed : [] };
+  if (!Array.isArray(parsed)) throw new Error('Invalid ledger: expected JSON array');
+  return { entries: parsed };
 }
 
 function _writeLedger(ledgerPath, entries) {
@@ -244,6 +245,73 @@ function _isLockStale(content, stat) {
   return mtimeAge > LOCK_ORPHAN_THRESHOLD_MS ? 'reclaimable' : 'wait';
 }
 
+// ── Metadata whitelist and reserved-field protection (INFRA-TASK-BE-03) ─────
+
+const METADATA_WHITELIST = new Set([
+  'agentId',
+  'nativeAgentName',
+  'platform',
+  'toolkitVersion',
+  'resolutionScope',
+  'definitionHash',
+]);
+
+const RESERVED_FIELDS = new Set([
+  'operation_id',
+  'agent',
+  'phase',
+  'model',
+  'status',
+  'started_at',
+  'completed_at',
+  'phase_delta_tokens',
+  'error',
+]);
+
+function _validateMetadata(metadata) {
+  for (const key of Object.keys(metadata)) {
+    if (RESERVED_FIELDS.has(key)) {
+      const err = new Error(
+        'open: metadata key "' + key + '" is a reserved field and cannot be set via metadata'
+      );
+      err.code = 'METADATA_RESERVED_FIELD';
+      err.key = key;
+      throw err;
+    }
+    if (!METADATA_WHITELIST.has(key)) {
+      const err = new Error(
+        'open: metadata key "' + key + '" is not in the whitelist; ' +
+        'allowed keys: ' + Array.from(METADATA_WHITELIST).join(', ')
+      );
+      err.code = 'METADATA_UNKNOWN_KEY';
+      err.key = key;
+      throw err;
+    }
+  }
+}
+
+function _extractStoredMetadata(entry) {
+  const result = {};
+  for (const key of METADATA_WHITELIST) {
+    if (Object.prototype.hasOwnProperty.call(entry, key)) {
+      result[key] = entry[key];
+    }
+  }
+  return result;
+}
+
+function _metadataEqual(storedEntry, newMetadata) {
+  const stored    = _extractStoredMetadata(storedEntry);
+  const storedKeys = Object.keys(stored).sort();
+  const newKeys    = Object.keys(newMetadata).sort();
+
+  if (storedKeys.length !== newKeys.length) return false;
+  for (var i = 0; i < newKeys.length; i++) {
+    if (stored[newKeys[i]] !== newMetadata[newKeys[i]]) return false;
+  }
+  return true;
+}
+
 // ── Public API (stubs — implemented by INFRA-TASK-BE-02 and US-0x tasks) ────
 
 function computeOperationId(prefix, agent, attempt) {
@@ -251,9 +319,14 @@ function computeOperationId(prefix, agent, attempt) {
   return crypto.createHash('sha256').update(input, 'utf8').digest('hex').slice(0, 32);
 }
 
-function open(dir, prefix, agent, phase, model, attempt) {
+function open(dir, prefix, agent, phase, model, attempt, metadata) {
   if (attempt == null) attempt = 1;
 
+  // Validate metadata keys BEFORE any lock or I/O (fail-closed).
+  if (metadata != null) {
+    _validateMetadata(metadata);
+  }
+
   const ledgerPath = path.join(dir, prefix + '-token-ledger.json');
   const lockPath   = ledgerPath + '.lock';
 
@@ -268,11 +341,25 @@ function open(dir, prefix, agent, phase, model, attempt) {
     });
 
     if (existingIndex !== -1) {
+      _assertUnowned(entries[existingIndex]);
+      // Idempotent re-open: check metadata conflict BEFORE writing anything.
+      if (metadata != null) {
+        if (!_metadataEqual(entries[existingIndex], metadata)) {
+          const err = new Error(
+            'open: metadata conflict for operation_id ' + operation_id +
+            ' — supplied metadata differs from stored metadata; re-open rejected (fail-closed)'
+          );
+          err.code = 'METADATA_CONFLICT';
+          err.operation_id = operation_id;
+          throw err;
+        }
+      }
       // Resume: idempotently preserve original started_at and any positive
       // phase_delta_tokens — only reset status to running.
       entries[existingIndex] = Object.assign({}, entries[existingIndex], { status: 'running' });
     } else {
-      entries.push({
+      // First open: store standard fields, then spread validated metadata fields.
+      const newEntry = {
         operation_id:       operation_id,
         agent:              agent,
         phase:              phase,
@@ -281,7 +368,11 @@ function open(dir, prefix, agent, phase, model, attempt) {
         started_at:         started_at,
         completed_at:       null,
         phase_delta_tokens: null,
-      });
+      };
+      if (metadata != null) {
+        Object.assign(newEntry, metadata);
+      }
+      entries.push(newEntry);
     }
 
     _writeLedger(ledgerPath, entries);
@@ -333,7 +424,8 @@ function close(dir, prefix, agent, tokens, attempt) {
       idx = agentIndexes[0];
     }
 
-    const existing     = entries[idx];
+    const existing = entries[idx];
+    _assertUnowned(existing);
     const completed_at = new Date().toISOString();
 
     // Token preservation (AC-10):
@@ -407,7 +499,8 @@ function fail(dir, prefix, agent, error, attempt) {
       idx = agentIndexes[0];
     }
 
-    const existing     = entries[idx];
+    const existing = entries[idx];
+    _assertUnowned(existing);
     const completed_at = new Date().toISOString();
 
     const update = {
@@ -483,7 +576,8 @@ function skip(dir, prefix, agent, phase, model, attempt) {
     }
 
     // Case 2: EXACTLY ONE match — update in place, preserving started_at and all other fields.
-    const existing     = entries[idx];
+    const existing = entries[idx];
+    _assertUnowned(existing);
     const completed_at = new Date().toISOString();
 
     entries[idx] = Object.assign({}, existing, {
@@ -500,11 +594,188 @@ function skip(dir, prefix, agent, phase, model, attempt) {
   }
 }
 
+// ── finalizeActivity (INFRA-TASK-BE-02) ──────────────────────────────────────
+//
+// Additive terminal-state API used by the executor. Unlike close()/fail()/skip(),
+// it requires an exact operation_id match (no name-only agent fallback), can
+// record tokens on ANY terminal status (including 'failed'), accepts an optional
+// usage_reason (additive ledger field, NOT an open() metadata whitelist key),
+// and is replay-safe: finalizing an already-terminal entry with identical data
+// is a no-op that returns the existing entry unchanged, while conflicting data
+// fails closed without writing. It does not introduce a second ledger file or
+// a duplicate token counter — it reuses phase_delta_tokens like close()/fail().
+
+const FINALIZE_STATUSES = new Set(['done', 'failed', 'skipped']);
+const TERMINAL_STATUSES = new Set(['done', 'failed', 'skipped']);
+
+function finalizeActivity(dir, prefix, agent, attempt, options) {
+  if (attempt == null) attempt = 1;
+  options = options || {};
+
+  const status        = options.status;
+  const tokens         = (options.tokens === undefined) ? null : options.tokens;
+  const reason         = options.reason;
+  const completedAtArg = options.completedAt;
+
+  if (!FINALIZE_STATUSES.has(status)) {
+    const err = new Error(
+      'finalizeActivity: status must be one of "done", "failed", "skipped"; got ' +
+      JSON.stringify(status)
+    );
+    err.code = 'INVALID_STATUS';
+    throw err;
+  }
+
+  const ledgerPath = path.join(dir, prefix + '-token-ledger.json');
+  const lockPath   = ledgerPath + '.lock';
+
+  const lockHandle = _acquireLock(lockPath);
+  try {
+    const { entries }  = _readLedger(ledgerPath);
+    const operation_id = computeOperationId(prefix, agent, attempt);
+
+    // Exact-ID lookup only — no name-only agent fallback (unlike close/fail/skip).
+    const idx = entries.findIndex(function (e) {
+      return e.operation_id === operation_id;
+    });
+
+    if (idx === -1) {
+      const err = new Error(
+        'finalizeActivity: no entry found for operation_id ' + operation_id +
+        ' in ' + ledgerPath + ' — exact operation_id match required, ' +
+        'no name-only fallback; operation was never opened'
+      );
+      err.code = 'ACTIVITY_NOT_FOUND';
+      err.operation_id = operation_id;
+      throw err;
+    }
+
+    const existing = entries[idx];
+    _assertUnowned(existing);
+
+    const suppliedTokens = (tokens === undefined) ? null : tokens;
+    const suppliedReason = (reason === undefined) ? null : reason;
+
+    if (TERMINAL_STATUSES.has(existing.status)) {
+      // Already finalized: this is either an idempotent replay of identical
+      // data (return unchanged, no write) or conflicting data (fail closed).
+      const existingTokens = (existing.phase_delta_tokens === undefined)
+        ? null : existing.phase_delta_tokens;
+      const existingReason = (existing.usage_reason === undefined)
+        ? null : existing.usage_reason;
+      const effectiveCompletedAt = (completedAtArg != null)
+        ? completedAtArg : existing.completed_at;
+
+      const isIdentical =
+        existing.status === status &&
+        existingTokens === suppliedTokens &&
+        existingReason === suppliedReason &&
+        existing.completed_at === effectiveCompletedAt;
+
+      if (isIdentical) {
+        return { status: 'ok', operation_id: operation_id, entry: existing, replay: true };
+      }
+
+      const err = new Error(
+        'finalizeActivity: conflicting terminal data for operation_id ' + operation_id +
+        ' in ' + ledgerPath + ' — existing terminal entry differs from the supplied ' +
+        'status/tokens/reason/completedAt; refusing to overwrite (fail-closed)'
+      );
+      err.code = 'TERMINAL_CONFLICT';
+      err.operation_id = operation_id;
+      throw err;
+    }
+
+    // Not yet terminal: finalize it. Tokens are recorded even when status is
+    // 'failed' — first finalization records known tokens, or null plus an
+    // optional usage_reason explaining why usage is unavailable. Matching the
+    // close() token-preservation invariant: a null/undefined supplied value
+    // never clobbers an already-stored positive phase_delta_tokens.
+    const completed_at = (completedAtArg != null) ? completedAtArg : new Date().toISOString();
+
+    let phase_delta_tokens = existing.phase_delta_tokens;
+    if (suppliedTokens !== null) {
+      phase_delta_tokens = suppliedTokens;
+    }
+
+    const update = {
+      status:             status,
+      completed_at:       completed_at,
+      phase_delta_tokens: phase_delta_tokens,
+    };
+    if (reason !== undefined) {
+      update.usage_reason = reason;
+    }
+
+    // Merge update: Object.assign preserves unknown/legacy fields and metadata verbatim.
+    entries[idx] = Object.assign({}, existing, update);
+
+    _writeLedger(ledgerPath, entries);
+
+    return { status: 'ok', operation_id: operation_id, entry: entries[idx] };
+  } finally {
+    _releaseLock(lockPath, lockHandle);
+  }
+}
+
+// Workflow-owned entries cannot be finalized by worker/legacy CLI calls.
+function _assertUnowned(entry) {
+  if (entry && entry.workflow_owner) {
+    const error = new Error('Workflow entry requires its owner capability');
+    error.code = 'LEDGER_OWNER_REQUIRED';
+    throw error;
+  }
+}
+
+function openOwned(dir, prefix, agent, phase, model, owner, metadata) {
+  if (typeof owner !== 'string' || !owner) throw new Error('Workflow owner required');
+  if (metadata) _validateMetadata(metadata);
+  const ledgerPath = path.join(dir, prefix + '-token-ledger.json');
+  const lockPath = ledgerPath + '.lock';
+  const handle = _acquireLock(lockPath);
+  try {
+    const { entries } = _readLedger(ledgerPath);
+    const operation_id = computeOperationId(prefix, agent, 1);
+    if (entries.some(e => e.operation_id === operation_id)) throw new Error('Owned operation already opened');
+    const entry = { operation_id, agent, phase, model, workflow_owner: owner,
+      status: 'running', started_at: new Date().toISOString(), completed_at: null, phase_delta_tokens: null };
+    if (metadata) Object.assign(entry, metadata);
+    entries.push(entry);
+    _writeLedger(ledgerPath, entries);
+    return entry;
+  } finally { _releaseLock(lockPath, handle); }
+}
+
+function finalizeOwned(dir, prefix, agent, owner, status, error, tokens) {
+  if (!['done', 'failed', 'skipped'].includes(status)) throw new Error('Invalid owned terminal status');
+  const ledgerPath = path.join(dir, prefix + '-token-ledger.json');
+  const lockPath = ledgerPath + '.lock';
+  const handle = _acquireLock(lockPath);
+  try {
+    const { entries } = _readLedger(ledgerPath);
+    const id = computeOperationId(prefix, agent, 1);
+    const entry = entries.find(e => e.operation_id === id);
+    if (!entry || !owner || entry.workflow_owner !== owner) throw new Error('Ledger ownership mismatch');
+    if (entry.status !== 'running') throw new Error('Owned terminal transition conflict');
+    if (tokens != null && (!Number.isInteger(tokens) || tokens <= 0)) throw new Error('Invalid owned token measurement');
+    if (tokens != null) entry.phase_delta_tokens = tokens;
+    entry.status = status;
+    entry.completed_at = new Date().toISOString();
+    if (error) entry.error = _sanitizeError(error);
+    _writeLedger(ledgerPath, entries);
+    return entry;
+  } finally { _releaseLock(lockPath, handle); }
+}
+
+
 module.exports = {
+  openOwned,
+  finalizeOwned,
   open,
   close,
   fail,
   skip,
+  finalizeActivity,
   computeOperationId,
   _readLedger,
   _writeLedger,
@@ -512,4 +783,9 @@ module.exports = {
   _acquireLock,
   _releaseLock,
   _isLockStale,
+  _validateMetadata,
+  _extractStoredMetadata,
+  _metadataEqual,
+  METADATA_WHITELIST,
+  RESERVED_FIELDS,
 };