@alipay/alipay-aipay 1.7.9 → 1.8.0

package/skill-pre/references/normal/scripts/telemetry.mjs

@@ -24,6 +24,8 @@ const syncLogCode = '1005';
 const syncLogMsg = 'alipay_aipay_skill_event';
 const syncLogYuyanID = '180020010001290755';
 const uploadTimeoutMs = 3000;
+const conversationUploadTimeoutMs = 10000;
+const uploadWorkerLifetimeMs = 60000;
 const permissionedForegroundFlushLimit = 5;
 const localWriteMaxAttempts = 2;
 const runStateLockWaitMs = 5000;
@@ -405,11 +407,12 @@ function writePrivateJson(file, value) {
   return false;
 }
 
-function acquireRunStateLock(env = process.env, lockPath = runStateLockPath(env)) {
+function acquireRunStateLock(env = process.env, lockPath = runStateLockPath(env), waitMs = runStateLockWaitMs) {
   const startedAt = Date.now();
+  let recoveredStaleLock = false;
   try {
     ensurePrivateDir(path.dirname(lockPath));
-    while (Date.now() - startedAt < runStateLockWaitMs) {
+    while (true) {
       try {
         const descriptor = fs.openSync(lockPath, 'wx', 0o600);
         fs.closeSync(descriptor);
@@ -418,8 +421,13 @@ function acquireRunStateLock(env = process.env, lockPath = runStateLockPath(env)
         if (error?.code !== 'EEXIST') return '';
         try {
           const lockStat = fs.statSync(lockPath);
-          if (Date.now() - lockStat.mtimeMs >= runStateLockStaleMs) fs.unlinkSync(lockPath);
+          if (!recoveredStaleLock && Date.now() - lockStat.mtimeMs >= runStateLockStaleMs) {
+            fs.unlinkSync(lockPath);
+            recoveredStaleLock = true;
+            continue;
+          }
         } catch {}
+        if (Date.now() - startedAt >= waitMs) return '';
         sleepSync(10);
       }
     }
@@ -427,8 +435,8 @@ function acquireRunStateLock(env = process.env, lockPath = runStateLockPath(env)
   return '';
 }
 
-function withRunStateLock(env = process.env, operation, requestedLockPath = runStateLockPath(env)) {
-  const lockPath = acquireRunStateLock(env, requestedLockPath);
+function withRunStateLock(env = process.env, operation, requestedLockPath = runStateLockPath(env), waitMs = runStateLockWaitMs) {
+  const lockPath = acquireRunStateLock(env, requestedLockPath, waitMs);
   if (!lockPath) return { acquired: false, value: null };
   try {
     return { acquired: true, value: operation() };
@@ -893,7 +901,7 @@ export function setConversationConsent({ sessionId = '', decision, inputId, conv
         }
       }
       return next;
-    }, conversationSessionFile(ownerHash, 'index.lock', env));
+    }, conversationSessionFile(ownerHash, 'index.lock', env), 0);
     return transaction.acquired ? transaction.value : null;
   } catch { return null; }
 }
@@ -920,6 +928,39 @@ function readConversationState(conversationId, env, { checkOwner = true } = {})
   } catch { return null; }
 }
 
+// Called only while holding the conversation's session lock. A private pending
+// marker survives a crash between enqueue and the final sequence/dedup commit.
+function reconcileConversationMessage(state, env) {
+  if (!state || !Object.prototype.hasOwnProperty.call(state, 'pending_message')) return state;
+  const pending = state.pending_message;
+  if (!pending || pending.sequence !== state.sequence + 1
+    || !Number.isSafeInteger(pending.sequence)
+    || !/^[a-f0-9]{32}$/.test(pending.message_hash || '')
+    || state.message_hashes.includes(pending.message_hash)
+    || !Number.isSafeInteger(pending.timestamp) || pending.timestamp <= 0) return null;
+  const basename = outboxFilename({ kind: 'conversation_message',
+    conversation_id: state.conversation_id, sequence: pending.sequence, timestamp: pending.timestamp });
+  let queued;
+  try {
+    const names = fs.readdirSync(outboxDir(env));
+    const name = names.find(name => name === basename || name.startsWith(`${basename}.sending.`));
+    if (name) {
+      queued = readOutboxRecord(path.join(outboxDir(env), name));
+      if (!queued || queued.event.conversation_id !== state.conversation_id
+        || queued.event.sequence !== pending.sequence || queued.event.timestamp !== pending.timestamp) return null;
+    }
+  } catch (error) {
+    // Missing queue means the process stopped before enqueue, or cleanup succeeded.
+    if (error?.code !== 'ENOENT') return null;
+  }
+  const { pending_message: _pending, ...next } = state;
+  if (queued) {
+    next.sequence = pending.sequence;
+    next.message_hashes = [...state.message_hashes, pending.message_hash];
+  }
+  return writePrivateJson(conversationStatePath(state.conversation_id, env), next) ? next : null;
+}
+
 export function startConversation({ sessionId = '' } = {}, env = process.env, nowMs = Date.now()) {
   if (!telemetryEnabled(env) || !conversationCode(env)) return null;
   try {
@@ -944,7 +985,7 @@ export function startConversation({ sessionId = '' } = {}, env = process.env, no
       if (!writePrivateJson(conversationStatePath(conversationId, env), state)
         || !writePrivateJson(indexPath, { conversation_id: conversationId })) return null;
       return { ...state, created: true };
-    }, `${indexPath}.lock`);
+    }, `${indexPath}.lock`, 0);
     return transaction.acquired ? transaction.value : null;
   } catch { return null; }
 }
@@ -966,7 +1007,7 @@ export function endConversation(conversationId, env = process.env) {
       if (!state || state.lifecycle === 'CLOSED') return state;
       const closed = { ...state, lifecycle: 'CLOSED' };
       return writePrivateJson(file, closed) ? closed : null;
-    }, conversationSessionFile(existing.session_hash, 'index.lock', env));
+    }, conversationSessionFile(existing.session_hash, 'index.lock', env), 0);
     return transaction.acquired ? transaction.value : null;
   } catch { return null; }
 }
@@ -992,7 +1033,7 @@ export async function recordConversationMessage(message, context = {}, env = pro
     const existing = readConversationState(conversationId, env);
     if (!existing) return null;
     const transaction = withRunStateLock(env, () => {
-      const state = readConversationState(conversationId, env);
+      const state = reconcileConversationMessage(readConversationState(conversationId, env), env);
       if (!state || state.lifecycle !== 'ACTIVE' || !hasConversationConsent(state, env)) return null;
       const hash = userInputHash(`${message.role}:${messageId}`, conversationId);
       const existingIndex = state.message_hashes.indexOf(hash);
@@ -1003,13 +1044,28 @@ export async function recordConversationMessage(message, context = {}, env = pro
       const event = normalizeConversationEvent({ ...conversationEventBase(state, productType, env, nowMs),
         sequence, role: message.role, content: sanitized.content, truncated: sanitized.truncated,
         has_attachments: message.has_attachments ?? false, redaction_version: 1 });
+      if (!event) return null;
+      // Reserve only recovery metadata, not a committed sequence or message body.
+      if (!writePrivateJson(file, { ...state, pending_message: {
+        sequence, message_hash: hash, timestamp: event.timestamp
+      } })) return null;
+      const queuedFile = writeOutboxRecord(event, env);
+      if (!queuedFile) {
+        writePrivateJson(file, state);
+        return null;
+      }
       const next = { ...state, sequence, message_hashes: [...state.message_hashes, hash] };
-      return event && writePrivateJson(file, next) ? event : null;
-    }, conversationSessionFile(existing.session_hash, 'index.lock', env));
+      if (!writePrivateJson(file, next)) {
+        try { fs.unlinkSync(queuedFile); } catch {}
+        return null;
+      }
+      return event;
+    }, conversationSessionFile(existing.session_hash, 'index.lock', env), 0);
     if (!transaction.acquired || !transaction.value) return null;
-    if (transaction.value.duplicate) return transaction.value;
+    // Recovered duplicate retries may also need to wake their queued upload.
+    // Already queued under the session lock; do not enqueue twice or wait for HTTP.
     // Never echo dialogue content in stderr markers, including in capture-only mode.
-    await deliverPersistedStageEvent(transaction.value, env, null);
+    if (telemetryUploadEnabled(env)) spawnOutboxFlush(env, { recordStageFailure: false });
     return transaction.value;
   } catch { return null; }
 }
@@ -1452,30 +1508,43 @@ export function isSuccessfulHttpStatus(statusCode) {
 }
 
 export function sendSyncLogReportDetailed(report, {
-  timeoutMs = report?.item?.msg === 'alipay_aipay_skill_conversation' ? 0 : uploadTimeoutMs
+  timeoutMs = report?.item?.msg === 'alipay_aipay_skill_conversation' ? conversationUploadTimeoutMs : uploadTimeoutMs,
+  request = https.request
 } = {}) {
   return new Promise((resolve) => {
     let settled = false;
+    let timer;
+    let req;
+    const limit = report?.item?.msg === 'alipay_aipay_skill_conversation' ? conversationUploadTimeoutMs : uploadTimeoutMs;
+    const budget = Number.isFinite(timeoutMs) && timeoutMs > 0 ? Math.min(timeoutMs, limit) : limit;
     const finish = (result) => {
       if (settled) return;
       settled = true;
+      clearTimeout(timer);
       resolve(result);
     };
+    const timeout = () => {
+      finish({ ok: false, error_code: 'ETIMEDOUT', error_message: 'request timeout' });
+      req?.destroy();
+    };
     try {
       if (!report?.item || !report?.options) {
         finish({ ok: false, error_code: 'INVALID_REPORT', error_message: 'invalid report' });
         return;
       }
       const body = syncLogRequestBody(report);
-      const req = https.request(`${syncLogServerURL}?biztype=${syncLogBizType}`, {
+      // A wall-clock deadline also covers DNS/TLS stalls and trickling responses.
+      timer = setTimeout(timeout, budget);
+      req = request(`${syncLogServerURL}?biztype=${syncLogBizType}`, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/x-www-form-urlencoded',
           'Content-Length': Buffer.byteLength(body)
         },
-        timeout: timeoutMs
+        timeout: budget
       }, (resp) => {
         resp.resume();
+        resp.on('error', error => finish(failedResult(error)));
         resp.on('end', () => {
           const statusCode = Number(resp.statusCode || 0);
           const ok = isSuccessfulHttpStatus(statusCode);
@@ -1489,10 +1558,7 @@ export function sendSyncLogReportDetailed(report, {
           });
         });
       });
-      req.on('timeout', () => {
-        finish({ ok: false, error_code: 'ETIMEDOUT', error_message: 'request timeout' });
-        req.destroy();
-      });
+      req.on('timeout', timeout);
       req.on('error', (error) => finish(failedResult(error)));
       req.end(body);
     } catch (error) {
@@ -1729,6 +1795,7 @@ export async function flushOutbox(env = process.env, {
   targetFile = '',
   timeoutMs = uploadTimeoutMs,
   sender = sendSyncLogReportDetailed,
+  onSent = null,
   prioritizeTerminal = false,
   includeConversations = true
 } = {}) {
@@ -1745,8 +1812,22 @@ export async function flushOutbox(env = process.env, {
     }
   }
   for (const file of files) {
-    const claimedFile = claimOutboxFile(file);
-    if (!claimedFile) continue;
+    const pending = file.endsWith('.conversation') ? readOutboxRecord(file) : null;
+    const owner = pending && readConversationState(pending.event.conversation_id, env, { checkOwner: false });
+    // Claim only after the recorder has committed state or removed its failed
+    // enqueue. Release this local lock before doing any network work.
+    const claim = owner ? withRunStateLock(env, () => {
+      const state = reconcileConversationMessage(
+        readConversationState(owner.conversation_id, env, { checkOwner: false }), env);
+      return state ? claimOutboxFile(file) : '';
+    }, conversationSessionFile(owner.session_hash, 'index.lock', env)) : null;
+    const claimedFile = owner ? (claim.acquired ? claim.value : '') : claimOutboxFile(file);
+    if (!claimedFile) {
+      // A local recovery/claim failure is retained work, not an empty batch.
+      // Let the worker stop instead of spinning on an uncommittable record.
+      if (fs.existsSync(file)) result.retained += 1;
+      continue;
+    }
     const record = readOutboxRecord(claimedFile);
     if (!record) {
       result.invalid += 1;
@@ -1756,7 +1837,7 @@ export async function flushOutbox(env = process.env, {
       continue;
     }
     if (isConversationEvent(record.event)) {
-      // Finish stage accounting before a conversation request can wait indefinitely.
+      // Finish stage accounting before handing off to conversation delivery.
       recordStageSuccess();
       // Legacy preview queues were captured before explicit consent existed.
       // Never retroactively authorize those messages with a later confirmation.
@@ -1766,6 +1847,13 @@ export async function flushOutbox(env = process.env, {
         result.invalid += 1;
         continue;
       }
+      // A recorder queues before committing its sequence. An already-running
+      // worker must leave that record pending until the state write succeeds.
+      if (record.event.sequence > state.sequence) {
+        releaseOutboxClaim(claimedFile);
+        result.retained += 1;
+        continue;
+      }
     }
     const report = buildSyncLogReport(
       record.event,
@@ -1785,7 +1873,7 @@ export async function flushOutbox(env = process.env, {
     let sendResult = { ok: false, error_code: 'SENDER_FAILED', error_message: 'sender failed' };
     try {
       const rawSendResult = await sender(report, {
-        timeoutMs: isConversationEvent(record.event) ? 0 : timeoutMs
+        timeoutMs: isConversationEvent(record.event) ? conversationUploadTimeoutMs : timeoutMs
       });
       sendResult = typeof rawSendResult === 'boolean' ? { ok: rawSendResult } : rawSendResult;
     } catch (error) {
@@ -1793,6 +1881,7 @@ export async function flushOutbox(env = process.env, {
     }
     if (sendResult?.ok) {
       result.sent += 1;
+      if (typeof onSent === 'function') onSent();
       if (!conversation) stages.sent += 1;
       try {
         fs.unlinkSync(claimedFile);
@@ -1826,7 +1915,7 @@ function outboxFilesForFlush(env = process.env, {
 } = {}) {
   const files = outboxFiles(env, { targetFile, limit: 0 })
     .filter((file) => includeConversations || !file.endsWith('.conversation'))
-    // Drain bounded stage uploads before potentially unbounded conversation requests.
+    // Drain stage uploads before the separately bounded conversation requests.
     .sort((a, b) => Number(a.endsWith('.conversation')) - Number(b.endsWith('.conversation')));
   if (!prioritizeTerminal || targetFile) return limit > 0 ? files.slice(0, limit) : files;
   const terminalFiles = [];
@@ -1840,11 +1929,139 @@ function outboxFilesForFlush(env = process.env, {
   return limit > 0 ? ordered.slice(0, limit) : ordered;
 }
 
+function uploadWorkerLeasePath(env) {
+  return path.join(outboxDir(env), '.upload-worker.json');
+}
+
+function readUploadWorkerLease(env) {
+  try {
+    const file = uploadWorkerLeasePath(env);
+    const lease = JSON.parse(fs.readFileSync(file, 'utf8'));
+    if (!Number.isSafeInteger(lease.pid) || lease.pid <= 0 || !validConversationId(lease.token)) return null;
+    // Older workers did not store a deadline. Bound their records by their
+    // original file age instead of trusting a reused PID indefinitely.
+    if (lease.started_at === undefined && lease.expires_at === undefined) {
+      const startedAt = Math.floor(fs.statSync(file).mtimeMs);
+      return { ...lease, started_at: startedAt,
+        expires_at: startedAt + uploadWorkerLifetimeMs + 2 * runStateLockWaitMs };
+    }
+    return lease;
+  } catch { return null; }
+}
+
+function uploadWorkerProcessAlive(lease) {
+  if (!lease) return false;
+  try { process.kill(lease.pid, 0); return true; }
+  catch (error) { return error.code !== 'ESRCH'; }
+}
+
+function uploadWorkerAlive(lease) {
+  if (!lease) return false;
+  const now = Date.now();
+  const maxAge = uploadWorkerLifetimeMs + 2 * runStateLockWaitMs;
+  if (!Number.isSafeInteger(lease.started_at) || !Number.isSafeInteger(lease.expires_at)
+    || lease.started_at <= 0 || lease.started_at > now || lease.expires_at <= now
+    || lease.expires_at <= lease.started_at || lease.expires_at - lease.started_at > maxAge) return false;
+  return uploadWorkerProcessAlive(lease);
+}
+
+function releaseWorkerClaims(env, pid = process.pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return;
+  try {
+    for (const name of fs.readdirSync(outboxDir(env))) {
+      if (isOutboxClaimName(name) && name.endsWith(`.sending.${pid}`)) {
+        releaseOutboxClaim(path.join(outboxDir(env), name));
+      }
+    }
+  } catch {}
+}
+
+async function runOutboxWorker(env) {
+  // Startup and exit share this short metadata lock; no network work holds it.
+  // Checking and replacing an expired/dead owner's lease are one operation.
+  const testLifetime = Number(env.ALIPAY_AIPAY_TEST_WORKER_LIFETIME_MS);
+  const lifetime = env.ALIPAY_AIPAY_TEST_MODE === '1' && Number.isFinite(testLifetime) && testLifetime > 0
+    ? Math.min(testLifetime, uploadWorkerLifetimeMs) : uploadWorkerLifetimeMs;
+  const transaction = withRunStateLock(env, () => {
+    const previous = readUploadWorkerLease(env);
+    if (uploadWorkerAlive(previous)) return null;
+    // A still-live reused PID may own a different foreground upload. Leave its
+    // claims to the existing TTL recovery instead of reclaiming active work.
+    if (previous && !uploadWorkerProcessAlive(previous)) releaseWorkerClaims(env, previous.pid);
+    const startedAt = Date.now();
+    const lease = { pid: process.pid, token: crypto.randomUUID(), started_at: startedAt,
+      expires_at: startedAt + lifetime + 2 * runStateLockWaitMs };
+    return writePrivateJson(uploadWorkerLeasePath(env), lease) ? lease : null;
+  }, `${uploadWorkerLeasePath(env)}.guard`);
+  const lease = transaction.acquired && transaction.value;
+  if (!lease) return;
+  let released = false;
+  let sent = 0;
+  let drained = false;
+  const cleanup = (keepIfPending = false) => {
+    if (released) return false;
+    const transaction = withRunStateLock(env, () => {
+      if (readUploadWorkerLease(env)?.token !== lease.token) { released = true; return false; }
+      // Foreground producers never wait for this guard. Check again after
+      // releasing it, so messages queued in the empty-check / exit window drain.
+      if (keepIfPending && outboxFilesForFlush(env, { limit: 1 }).length > 0) return true;
+      releaseWorkerClaims(env);
+      fs.unlinkSync(uploadWorkerLeasePath(env));
+      released = true;
+      return false;
+    }, `${uploadWorkerLeasePath(env)}.guard`);
+    return transaction.acquired && transaction.value;
+  };
+  // Production lifetime is fixed. Only the explicit test mode can shorten it.
+  const watchdog = setTimeout(() => {
+    cleanup();
+    // Continue a healthy backlog in a new bounded worker. A stalled first
+    // request never creates an endless chain of replacement processes.
+    if (released && sent > 0 && hasPendingOutboxRecords(env)) spawnOutboxFlush({ ...env,
+      ALIPAY_AIPAY_TELEMETRY_FLUSH_TARGET_FILE: '', ALIPAY_AIPAY_TELEMETRY_FLUSH_DELAY_MS: '0' });
+    process.exit(0);
+  }, lifetime);
+  try {
+    const delay = parseNonNegativeInteger(env.ALIPAY_AIPAY_TELEMETRY_FLUSH_DELAY_MS);
+    if (delay) await new Promise(resolve => setTimeout(resolve, Math.min(delay, 30000)));
+    const timeout = Number(env.ALIPAY_AIPAY_TELEMETRY_UPLOAD_TIMEOUT_MS || uploadTimeoutMs);
+    let targetFile = env.ALIPAY_AIPAY_TELEMETRY_FLUSH_TARGET_FILE || '';
+    while (!released) {
+      const result = await flushOutbox(env, {
+        targetFile,
+        timeoutMs: Number.isFinite(timeout) ? timeout : uploadTimeoutMs,
+        onSent: () => { sent += 1; }
+      });
+      targetFile = '';
+      // Do not busy-retry network or local failures. Keep their records queued.
+      if (result.retained > 0) break;
+      if (!cleanup(true)) { drained = true; break; }
+    }
+  } finally {
+    clearTimeout(watchdog);
+    cleanup();
+    if (drained && released && outboxFilesForFlush(env, { limit: 1 }).length > 0) {
+      spawnOutboxFlush({ ...env,
+        ALIPAY_AIPAY_TELEMETRY_FLUSH_TARGET_FILE: '', ALIPAY_AIPAY_TELEMETRY_FLUSH_DELAY_MS: '0' });
+    }
+  }
+}
+
 function spawnOutboxFlush(env = process.env, {
   delayMs = 0,
   recordStageFailure = hasPendingOutboxRecords(env, { includeConversations: false })
 } = {}) {
   if (!telemetryUploadEnabled(env)) return false;
+  // Best effort: a busy uploader must not delay capture or business delivery.
+  // Its post-release queue check handles an overlapping successful exit.
+  const transaction = withRunStateLock(env, () => {
+    if (uploadWorkerAlive(readUploadWorkerLease(env))) return true;
+    return launchOutboxFlush(env, { delayMs, recordStageFailure });
+  }, `${uploadWorkerLeasePath(env)}.guard`, 0);
+  return transaction.acquired && transaction.value;
+}
+
+function launchOutboxFlush(env, { delayMs, recordStageFailure }) {
   try {
     const childEnv = {
       ...env,
@@ -2066,13 +2283,7 @@ export async function runTelemetryCli(argv = process.argv) {
     if (command === 'flush-outbox-worker') {
       assertCliArgs(args, new Set(), 0);
       if (!telemetryUploadEnabled(process.env)) return 0;
-      const timeout = Number(process.env.ALIPAY_AIPAY_TELEMETRY_UPLOAD_TIMEOUT_MS || uploadTimeoutMs);
-      const delay = parseNonNegativeInteger(process.env.ALIPAY_AIPAY_TELEMETRY_FLUSH_DELAY_MS);
-      if (delay && delay > 0) sleepSync(Math.min(delay, 30000));
-      await flushOutbox(process.env, {
-        targetFile: process.env.ALIPAY_AIPAY_TELEMETRY_FLUSH_TARGET_FILE || '',
-        timeoutMs: Number.isFinite(timeout) ? timeout : uploadTimeoutMs
-      });
+      await runOutboxWorker(process.env);
       return 0;
     }
     if (!telemetryEnabled(process.env)) return 0;