@h-ai/reach 0.1.0-alpha.74 → 0.1.0-alpha.76

package/README.en.md

@@ -223,6 +223,8 @@ An empty successful 2xx response, including 204/205, means receiver acceptance;
 
 API `redirect` defaults to `error`; ordinary messages may explicitly use `follow`. Sensitive messages always reject redirects without forwarding payloads or private headers. `requireHttps:true` enforces initial HTTPS and cannot be combined with follow. When omitted, production sensitive messages require HTTPS; explicit false is only for an operator-approved trusted internal connection. Unknown source-side effects still require reconciliation and sensitive messages are never retried automatically. Application allowlists and network egress controls remain necessary.
 
+PostgreSQL/MySQL queue claims, renewals and final acknowledgements use the database execution clock inside SQL; batch cutoffs also come from the database. Node clock skew and row-lock waits cannot revive an expired owner. Local SQLite remains a development/test backend with a controllable test clock.
+
 ## Development and testing
 
 ```bash

package/README.ja.md

@@ -223,6 +223,8 @@ if (state.data?.status === 'dead' && state.data.updatedAt !== undefined) {
 
 API の `redirect` は既定で `error`、通常通知のみ明示的に `follow` を選択できます。sensitive は常に転送を拒否し、本文や独自ヘッダーを転送しません。`requireHttps:true` は開始URLの HTTPS を必須とし follow と併用できません。省略時は本番の機密通知に HTTPS を要求し、明示 false は運用者が確認した信頼できる内部接続専用です。元の送信先での副作用が不明なら確認が必要で、機密通知は自動再送しません。アプリの許可リストとネットワーク出口制御は引き続き必要です。
 
+PostgreSQL/MySQLの取得・更新・最終確認はSQL実行時のDB時計を使い、走査期限もDBから取得します。ノード時差や行ロック待機で期限切れ所有者を復活させません。SQLiteの制御可能な時計は開発・テスト専用です。
+
 ## 開発とテスト
 
 ```bash

package/README.md

@@ -223,6 +223,8 @@ if (state.data?.status === 'dead' && state.data.updatedAt !== undefined) {
 
 API Provider 的 `redirect` 默认为 `error`；只有普通消息可显式设为 `follow`。`sensitive:true` 始终拒绝跳转，不转发正文和私有头。`requireHttps:true` 强制初始 HTTPS 且与 follow 互斥；省略时生产敏感消息要求 HTTPS，显式 false 仅适用于操作者确认的受信内网。超时或源端已产生的未知副作用仍须核对，敏感消息不自动重投。域名授权、内网地址限制和出口策略由应用及部署环境负责。
 
+PostgreSQL/MySQL 队列的领取、续租和最终确认使用 SQL 执行时的数据库时钟；扫描截止也从数据库读取。节点时差及行锁等待跨过期限后不能续活旧 owner。本地 SQLite 只用于开发/测试，可控制测试时钟，不作为生产共享状态。
+
 ## 开发测试
 
 ```bash

package/dist/index.js

@@ -951,7 +951,12 @@ async function flushPendingMessages(providers2, repo) {
     }
   }
   try {
-    const eligibleBefore = Date.now();
+    const clock = await repo.currentTime();
+    if (!clock.success) {
+      logger6.warn("Failed to read queue clock", { errorCode: clock.error.code });
+      return;
+    }
+    const eligibleBefore = clock.data;
     for (let processed = 0; processed < FLUSH_BATCH_SIZE; processed += 1) {
       const claimResult = await repo.claimPendingBatch(flushOwner, {
         limit: 1,
@@ -1266,9 +1271,11 @@ function migrateReach(options = {}, db = reldb) {
 var TABLE_NAME = "hai_reach_send_log";
 var DEFAULT_CLAIM_BATCH_SIZE = 100;
 var DEFAULT_CLAIM_LOCK_MS = 10 * 60 * 1e3;
-var CLAIMABLE_STATUS_WHERE = "(attempt_count < max_attempts AND ((status = ? AND next_attempt_at <= ?) OR (status = ? AND updated_at < ? AND (locked_until IS NULL OR locked_until <= ?)) OR (status = ? AND updated_at < ? AND retry_safe = TRUE AND locked_until <= ?)))";
-function claimableParams(now, eligibleBefore = now) {
-  return ["pending", eligibleBefore, "processing", eligibleBefore, now, "sending", eligibleBefore, now];
+function claimableWhere(clock) {
+  return `(attempt_count < max_attempts AND ((status = ? AND next_attempt_at <= ?) OR (status = ? AND updated_at < ? AND (locked_until IS NULL OR locked_until <= ${clock})) OR (status = ? AND updated_at < ? AND retry_safe = TRUE AND locked_until <= ${clock})))`;
+}
+function claimableParams(eligibleBefore) {
+  return ["pending", eligibleBefore, "processing", eligibleBefore, "sending", eligibleBefore];
 }
 function buildSendLogError(detail, cause) {
   return err(
@@ -1479,6 +1486,18 @@ async function createSendLogRepository(db) {
   return ok(repo);
 }
 var DbSendLogRepository = class extends BaseReldbCrudRepository {
+  /** 共享后端取真正执行时钟；本地 SQLite 保留可控的单进程测试时钟。 */
+  clockSql(localNow) {
+    if (this.db.config?.type === "postgresql")
+      return "CAST(FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000) AS BIGINT)";
+    if (this.db.config?.type === "mysql")
+      return "CAST(UNIX_TIMESTAMP(SYSDATE(3)) * 1000 AS UNSIGNED)";
+    return String(localNow ?? Date.now());
+  }
+  async currentTime() {
+    const time = await this.sql().get(`SELECT ${this.clockSql()} AS now`);
+    return time.success && time.data && Number.isSafeInteger(Number(time.data.now)) ? ok(Number(time.data.now)) : buildSendLogError(reachM("reach_claimLost"));
+  }
   async settleSensitive(queueId, status, messageId) {
     const settled = await this.sql().execute(
       `UPDATE ${TABLE_NAME} SET status = ?, message_id = ?, locked_until = NULL, updated_at = ? WHERE queue_id = ? AND status = 'sending' AND body IS NULL AND vars_json IS NULL AND retry_safe = FALSE`,
@@ -1538,15 +1557,15 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
    * - 并发事务会直接跳过这些行，而不是重复命中同一批记录
    * - 在 commit 前不会暴露给其它节点重复 claim
    */
-  async claimPendingIdsWithRowLocks(tx, owner, now, lockedUntil, limit, eligibleBefore) {
+  async claimPendingIdsWithRowLocks(tx, owner, clock, lockMs, limit, eligibleBefore) {
     const lockedRowsResult = await tx.query(
       `SELECT id
        FROM ${TABLE_NAME}
-       WHERE ${CLAIMABLE_STATUS_WHERE}
+       WHERE ${claimableWhere(clock)}
        ORDER BY created_at ASC
        LIMIT ?
        FOR UPDATE SKIP LOCKED`,
-      [...claimableParams(now, eligibleBefore), limit]
+      [...claimableParams(eligibleBefore), limit]
     );
     if (!lockedRowsResult.success) {
       throw lockedRowsResult.error;
@@ -1564,9 +1583,9 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
     const placeholders = claimedIds.map(() => "?").join(", ");
     const updateResult = await tx.execute(
       `UPDATE ${TABLE_NAME}
-       SET status = ?, processing_owner = ?, locked_until = ?, updated_at = ?, attempt_count = attempt_count + 1
+       SET status = ?, processing_owner = ?, locked_until = ${clock} + ?, updated_at = ${clock}, attempt_count = attempt_count + 1
        WHERE id IN (${placeholders})`,
-      ["processing", owner, lockedUntil, now, ...claimedIds]
+      ["processing", owner, lockMs, ...claimedIds]
     );
     if (!updateResult.success) {
       throw updateResult.error;
@@ -1580,10 +1599,10 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
    * 即使多个节点读到了同一候选，只有第一个满足条件的 UPDATE 会成功，
    * 从而避免同一行被两个节点同时 claim。
    */
-  async claimPendingIdsWithConditionalUpdates(tx, owner, now, lockedUntil, limit, eligibleBefore) {
+  async claimPendingIdsWithConditionalUpdates(tx, owner, clock, lockMs, limit, eligibleBefore) {
     const candidatesResult = await this.findAll({
-      where: CLAIMABLE_STATUS_WHERE,
-      params: claimableParams(now, eligibleBefore),
+      where: claimableWhere(clock),
+      params: claimableParams(eligibleBefore),
       orderBy: "created_at ASC",
       limit
     }, tx);
@@ -1594,9 +1613,9 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
     for (const row of candidatesResult.data) {
       const updateResult = await tx.execute(
         `UPDATE ${TABLE_NAME}
-         SET status = ?, processing_owner = ?, locked_until = ?, updated_at = ?, attempt_count = attempt_count + 1
-         WHERE id = ? AND ${CLAIMABLE_STATUS_WHERE}`,
-        ["processing", owner, lockedUntil, now, row.id, ...claimableParams(now, eligibleBefore)]
+         SET status = ?, processing_owner = ?, locked_until = ${clock} + ?, updated_at = ${clock}, attempt_count = attempt_count + 1
+         WHERE id = ? AND ${claimableWhere(clock)}`,
+        ["processing", owner, lockMs, row.id, ...claimableParams(eligibleBefore)]
       );
       if (!updateResult.success) {
         throw updateResult.error;
@@ -1622,10 +1641,13 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
   }
   /** claim 一批待发送记录，避免多节点重复发送 */
   async claimPendingBatch(owner, options) {
-    const now = options?.now ?? Date.now();
+    const time = await this.currentTime();
+    if (!time.success)
+      return time;
+    const now = this.db.config?.type === "sqlite" ? options?.now ?? time.data : time.data;
+    const clock = this.clockSql(now);
     const limit = options?.limit ?? DEFAULT_CLAIM_BATCH_SIZE;
     const lockMs = options?.lockMs ?? DEFAULT_CLAIM_LOCK_MS;
-    const lockedUntil = now + lockMs;
     const eligibleBefore = Math.min(now, options?.eligibleBefore ?? now);
     const dbType = this.db.config?.type;
     const txResult = await this.db.tx.begin();
@@ -1635,23 +1657,23 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
     const tx = txResult.data;
     try {
       const unknown = await tx.execute(
-        `UPDATE ${TABLE_NAME} SET status = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ?
-         WHERE status = ? AND retry_safe = FALSE AND (locked_until IS NULL OR locked_until <= ?)`,
-        ["unknown", now, "sending", now]
+        `UPDATE ${TABLE_NAME} SET status = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ${clock}
+         WHERE status = ? AND retry_safe = FALSE AND (locked_until IS NULL OR locked_until <= ${clock})`,
+        ["unknown", "sending"]
       );
       if (!unknown.success)
         throw unknown.error;
       const exhausted = await tx.execute(
         `UPDATE ${TABLE_NAME}
-         SET status = ?, last_failure = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ?
+         SET status = ?, last_failure = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ${clock}
          WHERE attempt_count >= max_attempts
-           AND (status = ? OR (status = ? AND (locked_until IS NULL OR locked_until <= ?))
-             OR (status = ? AND retry_safe = TRUE AND locked_until <= ?))`,
-        ["dead", "attempts_exhausted", now, "pending", "processing", now, "sending", now]
+           AND (status = ? OR (status = ? AND (locked_until IS NULL OR locked_until <= ${clock}))
+             OR (status = ? AND retry_safe = TRUE AND locked_until <= ${clock}))`,
+        ["dead", "attempts_exhausted", "pending", "processing", "sending"]
       );
       if (!exhausted.success)
         throw exhausted.error;
-      const claimedIds = dbType === "postgresql" || dbType === "mysql" ? await this.claimPendingIdsWithRowLocks(tx, owner, now, lockedUntil, limit, eligibleBefore) : await this.claimPendingIdsWithConditionalUpdates(tx, owner, now, lockedUntil, limit, eligibleBefore);
+      const claimedIds = dbType === "postgresql" || dbType === "mysql" ? await this.claimPendingIdsWithRowLocks(tx, owner, clock, lockMs, limit, eligibleBefore) : await this.claimPendingIdsWithConditionalUpdates(tx, owner, clock, lockMs, limit, eligibleBefore);
       if (claimedIds.length === 0) {
         const commitResult2 = await tx.commit();
         if (!commitResult2.success) {
@@ -1679,15 +1701,15 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
   }
   /** 失败提交与 owner、次数和有效租约同时比较，旧 worker 不能更改新领取的退避策略。 */
   async failClaim(row, owner, reason, permanent = false, tx) {
-    const now = Date.now();
+    const clock = this.clockSql();
     const dead = permanent || row.attemptCount >= row.maxAttempts;
     const delay = Math.min(row.maxRetryDelayMs, row.retryDelayMs * 2 ** Math.min(row.attemptCount - 1, 30));
     const result = await this.sql(tx).execute(
       `UPDATE ${TABLE_NAME}
-       SET status = ?, last_failure = ?, next_attempt_at = ?, processing_owner = NULL, locked_until = NULL, updated_at = ?
-       WHERE id = ? AND processing_owner = ? AND locked_until > ? AND attempt_count = ?
+       SET status = ?, last_failure = ?, next_attempt_at = ${dead ? "0" : `${clock} + ?`}, processing_owner = NULL, locked_until = NULL, updated_at = ${clock}
+       WHERE id = ? AND processing_owner = ? AND locked_until > ${clock} AND attempt_count = ?
          AND (status = ? OR (status = ? AND retry_safe = TRUE))`,
-      [dead ? "dead" : "pending", reason, dead ? 0 : now + delay, now, row.id, owner, now, row.attemptCount, "processing", "sending"]
+      [dead ? "dead" : "pending", reason, ...dead ? [] : [delay], row.id, owner, row.attemptCount, "processing", "sending"]
     );
     if (!result.success) {
       return buildSendLogError(result.error.message, result.error);
@@ -1699,60 +1721,60 @@ var DbSendLogRepository = class extends BaseReldbCrudRepository {
   }
   /** 仅当前 dead 版本可以恢复；保留 queueId，迟到的幂等 API 请求仍使用同一键。 */
   async requeueDead(queueId, expectedUpdatedAt) {
-    const now = Math.max(Date.now(), expectedUpdatedAt + 1);
+    const clock = this.clockSql();
     const result = await this.sql().execute(
-      `UPDATE ${TABLE_NAME} SET status = ?, attempt_count = 0, next_attempt_at = ?, last_failure = NULL, updated_at = ?
+      `UPDATE ${TABLE_NAME} SET status = ?, attempt_count = 0, next_attempt_at = ${clock}, last_failure = NULL, updated_at = CASE WHEN updated_at >= ${clock} THEN updated_at + 1 ELSE ${clock} END
        WHERE queue_id = ? AND status = ? AND updated_at = ?`,
-      ["pending", now, now, queueId, "dead", expectedUpdatedAt]
+      ["pending", queueId, "dead", expectedUpdatedAt]
     );
     return result.success && result.data.changes === 1 ? ok(void 0) : buildSendLogError(reachM("reach_deadRequeueFailed"));
   }
   /** 发送前持久化门闩；仅有效租约可从 processing 进入 sending。 */
   async beginDelivery(id, owner, retrySafe, lockMs = DEFAULT_CLAIM_LOCK_MS) {
-    const now = Date.now();
+    const clock = this.clockSql();
     const flag = this.db.config?.type === "postgresql" ? retrySafe : Number(retrySafe);
     const result = await this.sql().execute(
-      `UPDATE ${TABLE_NAME} SET status = ?, retry_safe = ?, locked_until = ?, updated_at = ?
-       WHERE id = ? AND status = ? AND processing_owner = ? AND locked_until > ?`,
-      ["sending", flag, now + lockMs, now, id, "processing", owner, now]
+      `UPDATE ${TABLE_NAME} SET status = ?, retry_safe = ?, locked_until = ${clock} + ?, updated_at = ${clock}
+       WHERE id = ? AND status = ? AND processing_owner = ? AND locked_until > ${clock}`,
+      ["sending", flag, lockMs, id, "processing", owner]
     );
     return result.success && result.data.changes === 1 ? ok(void 0) : buildSendLogError(reachM("reach_claimLost"));
   }
   /** 续租不接受已经过期的 owner，防止暂停的旧进程恢复后重新取得发送权。 */
   async renewClaim(id, owner, lockMs = DEFAULT_CLAIM_LOCK_MS) {
-    const now = Date.now();
+    const clock = this.clockSql();
     const result = await this.sql().execute(
-      `UPDATE ${TABLE_NAME} SET locked_until = ?, updated_at = ?
-       WHERE id = ? AND processing_owner = ? AND status IN (?, ?) AND locked_until > ?`,
-      [now + lockMs, now, id, owner, "processing", "sending", now]
+      `UPDATE ${TABLE_NAME} SET locked_until = ${clock} + ?, updated_at = ${clock}
+       WHERE id = ? AND processing_owner = ? AND status IN (?, ?) AND locked_until > ${clock}`,
+      [lockMs, id, owner, "processing", "sending"]
     );
     return result.success && result.data.changes === 1 ? ok(void 0) : buildSendLogError(reachM("reach_claimLost"));
   }
   /** 未知结果只能停止自动处理，不推断外部副作用已撤销。 */
   async markUnknown(id, owner) {
     const result = await this.sql().execute(
-      `UPDATE ${TABLE_NAME} SET status = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ?
+      `UPDATE ${TABLE_NAME} SET status = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ${this.clockSql()}
        WHERE id = ? AND processing_owner = ? AND status IN (?, ?)`,
-      ["unknown", Date.now(), id, owner, "processing", "sending"]
+      ["unknown", id, owner, "processing", "sending"]
     );
     return result.success && result.data.changes === 1 ? ok(void 0) : buildSendLogError(reachM("reach_claimLost"));
   }
   /** 人工裁定的条件提交，不提供重新排队或重新发送操作。 */
   async resolveUnknown(queueId, resolution) {
     const result = await this.sql().execute(
-      `UPDATE ${TABLE_NAME} SET status = ?, message_id = ?, next_attempt_at = 0, updated_at = ? WHERE queue_id = ? AND status = ?`,
-      [resolution.status, resolution.messageId ?? null, Date.now(), queueId, "unknown"]
+      `UPDATE ${TABLE_NAME} SET status = ?, message_id = ?, next_attempt_at = 0, updated_at = ${this.clockSql()} WHERE queue_id = ? AND status = ?`,
+      [resolution.status, resolution.messageId ?? null, queueId, "unknown"]
     );
     return result.success && result.data.changes === 1 ? ok(void 0) : buildSendLogError(reachM("reach_unknownResolutionFailed"));
   }
   /** 只有仍持有 sending 有效租约的 owner 可以确认投递。 */
   async markSent(id, owner, messageId, tx) {
-    const now = Date.now();
+    const clock = this.clockSql();
     const result = await this.sql(tx).execute(
       `UPDATE ${TABLE_NAME}
-       SET status = ?, message_id = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ?
-       WHERE id = ? AND status = ? AND processing_owner = ? AND locked_until > ?`,
-      ["sent", messageId ?? null, now, id, "sending", owner, now]
+       SET status = ?, message_id = ?, processing_owner = NULL, locked_until = NULL, next_attempt_at = 0, updated_at = ${clock}
+       WHERE id = ? AND status = ? AND processing_owner = ? AND locked_until > ${clock}`,
+      ["sent", messageId ?? null, id, "sending", owner]
     );
     if (!result.success) {
       return buildSendLogError(result.error.message, result.error);