@powerhousedao/ph-cli 6.0.0-dev.90 → 6.0.0-dev.91

package/dist/src/cli.js

@@ -332315,6 +332315,7 @@ class GqlRequestChannel {
   cursorStorage;
   operationIndex;
   pollTimer;
+  abortController = new AbortController;
   isShutdown;
   failureCount;
   lastSuccessUtcMs;
@@ -332395,6 +332396,7 @@ class GqlRequestChannel {
     });
   }
   shutdown() {
+    this.abortController.abort();
     this.bufferedOutbox.flush();
     this.isShutdown = true;
     this.pollTimer.stop();
@@ -332422,7 +332424,7 @@ class GqlRequestChannel {
     };
   }
   async init() {
-    await this.touchRemoteChannel();
+    const { ackOrdinal } = await this.touchRemoteChannel();
     const cursors = await this.cursorStorage.list(this.remoteName);
     const inboxOrdinal = cursors.find((c2) => c2.cursorType === "inbox")?.cursorOrdinal ?? 0;
     const outboxOrdinal = cursors.find((c2) => c2.cursorType === "outbox")?.cursorOrdinal ?? 0;
@@ -332430,6 +332432,9 @@ class GqlRequestChannel {
     this.outbox.init(outboxOrdinal);
     this.lastPersistedInboxOrdinal = inboxOrdinal;
     this.lastPersistedOutboxOrdinal = outboxOrdinal;
+    if (ackOrdinal > 0) {
+      trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
+    }
     this.pollTimer.setDelegate(() => this.poll());
     this.pollTimer.start();
     this.transitionConnectionState("connected");
@@ -332499,34 +332504,57 @@ class GqlRequestChannel {
     this.deadLetter.add(...syncOps);
   }
   handlePollError(error48) {
+    if (this.isShutdown)
+      return true;
     const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
     if (err2.message.includes("Channel not found")) {
       this.transitionConnectionState("reconnecting");
       this.recoverFromChannelNotFound();
       return true;
     }
+    const classification = this.classifyError(err2);
     this.failureCount++;
     this.lastFailureUtcMs = Date.now();
-    this.transitionConnectionState("error");
     const channelError = new ChannelError("inbox", err2);
-    this.logger.error("GqlChannel poll error (@FailureCount): @Error", this.failureCount, channelError);
+    this.logger.error("GqlChannel poll error (@FailureCount, @Classification): @Error", this.failureCount, classification, channelError);
+    if (classification === "unrecoverable") {
+      this.pollTimer.stop();
+      this.transitionConnectionState("error");
+      return true;
+    }
+    this.transitionConnectionState("error");
     return false;
   }
   recoverFromChannelNotFound() {
     this.logger.info("GqlChannel @ChannelId not found on remote, re-registering...", this.channelId);
     this.pollTimer.stop();
-    this.touchRemoteChannel().then(() => {
-      this.logger.info("GqlChannel @ChannelId re-registered successfully", this.channelId);
-      this.failureCount = 0;
-      this.pollTimer.start();
-      this.transitionConnectionState("connected");
-    }).catch((recoveryError) => {
-      this.logger.error("GqlChannel @ChannelId failed to re-register: @Error", this.channelId, recoveryError);
-      this.failureCount++;
-      this.lastFailureUtcMs = Date.now();
-      this.pollTimer.start();
-      this.transitionConnectionState("error");
-    });
+    const attemptRecovery = (attempt) => {
+      if (this.isShutdown)
+        return;
+      this.touchRemoteChannel().then(({ ackOrdinal }) => {
+        this.logger.info("GqlChannel @ChannelId re-registered successfully", this.channelId);
+        this.failureCount = 0;
+        if (ackOrdinal > 0) {
+          trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
+        }
+        this.pollTimer.start();
+        this.transitionConnectionState("connected");
+      }).catch((recoveryError) => {
+        const err2 = recoveryError instanceof Error ? recoveryError : new Error(String(recoveryError));
+        const classification = this.classifyError(err2);
+        this.logger.error("GqlChannel @ChannelId recovery attempt @Attempt failed (@Classification): @Error", this.channelId, attempt, classification, recoveryError);
+        this.failureCount++;
+        this.lastFailureUtcMs = Date.now();
+        if (classification === "unrecoverable") {
+          this.transitionConnectionState("error");
+          return;
+        }
+        this.transitionConnectionState("reconnecting");
+        const delay = calculateBackoffDelay(attempt, this.config.retryBaseDelayMs, this.config.retryMaxDelayMs, Math.random());
+        setTimeout(() => attemptRecovery(attempt + 1), delay);
+      });
+    };
+    attemptRecovery(1);
   }
   async pollSyncEnvelopes(ackOrdinal, latestOrdinal) {
     const query = `
@@ -332624,7 +332652,10 @@ class GqlRequestChannel {
     } catch {}
     const mutation = `
       mutation TouchChannel($input: TouchChannelInput!) {
-        touchChannel(input: $input)
+        touchChannel(input: $input) {
+          success
+          ackOrdinal
+        }
       }
     `;
     const variables = {
@@ -332640,7 +332671,11 @@ class GqlRequestChannel {
         sinceTimestampUtcMs
       }
     };
-    await this.executeGraphQL(mutation, variables);
+    const data = await this.executeGraphQL(mutation, variables);
+    if (!data.touchChannel.success) {
+      throw new GraphQLRequestError("touchChannel returned success=false", "graphql");
+    }
+    return { ackOrdinal: data.touchChannel.ackOrdinal };
   }
   attemptPush(syncOps) {
     this.pushSyncOperations(syncOps).then(() => {
@@ -332650,8 +332685,11 @@ class GqlRequestChannel {
         this.transitionConnectionState("connected");
       }
     }).catch((error48) => {
+      if (this.isShutdown)
+        return;
       const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
-      if (this.isRecoverablePushError(err2)) {
+      const classification = this.classifyError(err2);
+      if (classification === "recoverable") {
         this.pushFailureCount++;
         this.pushBlocked = true;
         this.logger.error("GqlChannel push failed (attempt @FailureCount), will retry: @Error", this.pushFailureCount, err2);
@@ -332685,12 +332723,26 @@ class GqlRequestChannel {
       this.attemptPush([...allItems]);
     }, delay);
   }
-  isRecoverablePushError(error48) {
-    if (error48.message.startsWith("GraphQL errors:"))
-      return false;
-    if (error48.message === "GraphQL response missing data field")
-      return false;
-    return true;
+  classifyError(error48) {
+    if (!(error48 instanceof GraphQLRequestError)) {
+      return "recoverable";
+    }
+    switch (error48.category) {
+      case "network":
+        return "recoverable";
+      case "http": {
+        if (error48.statusCode !== undefined && error48.statusCode >= 500) {
+          return "recoverable";
+        }
+        return "unrecoverable";
+      }
+      case "parse":
+        return "recoverable";
+      case "graphql":
+        return "unrecoverable";
+      case "missing-data":
+        return "unrecoverable";
+    }
   }
   async pushSyncOperations(syncOps) {
     for (const syncOp of syncOps) {
@@ -332767,26 +332819,27 @@ class GqlRequestChannel {
         body: JSON.stringify({
           query,
           variables
-        })
+        }),
+        signal: this.abortController.signal
       });
     } catch (error48) {
-      throw new Error(`GraphQL request failed: ${error48 instanceof Error ? error48.message : String(error48)}`);
+      throw new GraphQLRequestError(`GraphQL request failed: ${error48 instanceof Error ? error48.message : String(error48)}`, "network");
     }
     if (!response.ok) {
-      throw new Error(`GraphQL request failed: ${response.status} ${response.statusText}`);
+      throw new GraphQLRequestError(`GraphQL request failed: ${response.status} ${response.statusText}`, "http", response.status);
     }
     let result;
     try {
       result = await response.json();
     } catch (error48) {
-      throw new Error(`Failed to parse GraphQL response: ${error48 instanceof Error ? error48.message : String(error48)}`);
+      throw new GraphQLRequestError(`Failed to parse GraphQL response: ${error48 instanceof Error ? error48.message : String(error48)}`, "parse");
     }
     this.logger.verbose("GQL response @channelId @operation status=@status data=@data errors=@errors", this.channelId, operationName, response.status, JSON.stringify(result.data), result.errors ? JSON.stringify(result.errors) : "none");
     if (result.errors) {
-      throw new Error(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`);
+      throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql");
     }
     if (!result.data) {
-      throw new Error("GraphQL response missing data field");
+      throw new GraphQLRequestError("GraphQL response missing data field", "missing-data");
     }
     return result.data;
   }
@@ -333337,6 +333390,7 @@ class SyncManager {
   remotes;
   awaiter;
   syncAwaiter;
+  abortController = new AbortController;
   isShutdown;
   eventUnsubscribe;
   failedEventUnsubscribe;
@@ -333397,6 +333451,7 @@ class SyncManager {
   }
   shutdown() {
     this.isShutdown = true;
+    this.abortController.abort();
     this.batchAggregator.clear();
     if (this.eventUnsubscribe) {
       this.eventUnsubscribe();
@@ -333589,6 +333644,8 @@ class SyncManager {
     return Array.from(this.remotes.values()).filter((remote) => remote.collectionId === collectionId);
   }
   async processCompleteBatch(batch) {
+    if (this.isShutdown)
+      return;
     const collectionIds = [
       ...new Set(Object.values(batch.collectionMemberships).flatMap((collections) => collections))
     ];
@@ -333632,8 +333689,10 @@ class SyncManager {
     const operations2 = syncOp.operations.map((op) => op.operation);
     let jobInfo;
     try {
-      jobInfo = await this.reactor.load(syncOp.documentId, syncOp.branch, operations2, undefined, { sourceRemote: remote.name });
+      jobInfo = await this.reactor.load(syncOp.documentId, syncOp.branch, operations2, this.abortController.signal, { sourceRemote: remote.name });
     } catch (error48) {
+      if (this.isShutdown)
+        return;
       const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
       this.logger.error("Failed to load operations from inbox (@remote, @documentId, @error)", remote.name, syncOp.documentId, err2.message);
       const channelError = new ChannelError("inbox", err2);
@@ -333644,8 +333703,10 @@ class SyncManager {
     }
     let completedJobInfo;
     try {
-      completedJobInfo = await this.awaiter.waitForJob(jobInfo.id);
+      completedJobInfo = await this.awaiter.waitForJob(jobInfo.id, this.abortController.signal);
     } catch (error48) {
+      if (this.isShutdown)
+        return;
       const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
       this.logger.error("Failed to wait for job completion (@remote, @documentId, @jobId, @error)", remote.name, syncOp.documentId, jobInfo.id, err2.message);
       const channelError = new ChannelError("inbox", err2);
@@ -333680,10 +333741,10 @@ class SyncManager {
     const request = { jobs };
     let result;
     try {
-      result = await this.reactor.loadBatch(request, undefined, {
-        sourceRemote
-      });
+      result = await this.reactor.loadBatch(request, this.abortController.signal, { sourceRemote });
     } catch (error48) {
+      if (this.isShutdown)
+        return;
       for (const { remote, syncOp } of items) {
         const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
         syncOp.failed(new ChannelError("inbox", err2));
@@ -333704,8 +333765,10 @@ class SyncManager {
       const jobInfo = result.jobs[syncOp.jobId];
       let completedJobInfo;
       try {
-        completedJobInfo = await this.awaiter.waitForJob(jobInfo.id);
+        completedJobInfo = await this.awaiter.waitForJob(jobInfo.id, this.abortController.signal);
       } catch (error48) {
+        if (this.isShutdown)
+          continue;
         const err2 = error48 instanceof Error ? error48 : new Error(String(error48));
         syncOp.failed(new ChannelError("inbox", err2));
         remote.channel.deadLetter.add(syncOp);
@@ -333752,7 +333815,7 @@ class SyncManager {
     remote.channel.outbox.add(...syncOps);
   }
   async getOperationsForRemote(remote, ackOrdinal) {
-    const results = await this.operationIndex.find(remote.collectionId, ackOrdinal, { excludeSourceRemote: remote.name });
+    const results = await this.operationIndex.find(remote.collectionId, ackOrdinal, { excludeSourceRemote: remote.name }, undefined, this.abortController.signal);
     let operations2 = results.results.map((entry) => toOperationWithContext(entry));
     const sinceTimestamp = remote.options.sinceTimestampUtcMs;
     if (sinceTimestamp && sinceTimestamp !== "0") {
@@ -333769,7 +333832,7 @@ var __defProp2, __export2 = (target, all) => {
       configurable: true,
       set: (newValue) => all[name4] = () => newValue
     });
-}, byteToHex, rnds8, randomUUID, PropagationMode, RelationshipChangeType, JobStatus, DocumentChangeType, AlterTableNode2, IdentifierNode2, CreateIndexNode2, CreateSchemaNode2, ON_COMMIT_ACTIONS2, CreateTableNode2, SchemableIdentifierNode2, DropIndexNode2, DropSchemaNode2, DropTableNode2, AliasNode2, TableNode2, SelectModifierNode2, AndNode2, OrNode2, OnNode2, JoinNode2, BinaryOperationNode2, COMPARISON_OPERATORS2, ARITHMETIC_OPERATORS2, JSON_OPERATORS2, BINARY_OPERATORS2, UNARY_FILTER_OPERATORS2, UNARY_OPERATORS2, OPERATORS2, OperatorNode2, ColumnNode2, SelectAllNode2, ReferenceNode2, OrderByItemNode2, RawNode2, CollateNode2, LOGGED_MESSAGES2, JSONReferenceNode2, JSONOperatorChainNode2, JSONPathNode2, PrimitiveValueListNode2, ValueListNode2, ValueNode2, ParensNode2, OrderByNode2, PartitionByNode2, OverNode2, FromNode2, GroupByNode2, HavingNode2, InsertQueryNode2, ListNode2, UpdateQueryNode2, UsingNode2, DeleteQueryNode2, WhereNode2, ReturningNode2, ExplainNode2, WhenNode2, MergeQueryNode2, OutputNode2, QueryNode2, SelectQueryNode2, PartitionByItemNode2, SelectionNode2, ValuesNode2, DefaultInsertValueNode2, ColumnUpdateNode2, OnDuplicateKeyNode2, NoResultError2, OnConflictNode2, TopNode2, OrActionNode2, LimitNode2, CommonTableExpressionNameNode2, CommonTableExpressionNode2, WithNode2, CHARS2, ROOT_OPERATION_NODES2, SCHEMALESS_FUNCTIONS2, WithSchemaTransformer2, MatchedNode2, NO_PLUGINS2, NoopQueryExecutor2, NOOP_QUERY_EXECUTOR2, OffsetNode2, GroupByItemNode2, SetOperationNode2, FetchNode2, AggregateFunctionNode2, FunctionNode2, UnaryOperationNode2, CaseNode2, JSONPathLegNode2, TraversedJSONPathBuilder2, TupleNode2, SIMPLE_COLUMN_DATA_TYPES2, COLUMN_DATA_TYPE_REGEX2, DataTypeNode2, CastNode2, AddColumnNode2, ColumnDefinitionNode2, DropColumnNode2, RenameColumnNode2, CheckConstraintNode2, ON_MODIFY_FOREIGN_ACTIONS2, ReferencesNode2, GeneratedNode2, DefaultValueNode2, ModifyColumnNode2, ForeignKeyConstraintNode2, AddConstraintNode2, UniqueConstraintNode2, DropConstraintNode2, AlterColumnNode2, PrimaryKeyConstraintNode2, AddIndexNode2, RenameConstraintNode2, ImmediateValueTransformer2, CreateViewNode2, DropViewNode2, CreateTypeNode2, DropTypeNode2, RefreshMaterializedViewNode2, DefaultQueryExecutor2, ignoreError2 = () => {}, TRANSACTION_ACCESS_MODES2, TRANSACTION_ISOLATION_LEVELS2, logLevels2, LOG_LEVELS4, Kysely2, Transaction2, ControlledTransaction2, sql3, LIT_WRAP_REGEX2, DefaultQueryCompiler2, SELECT_MODIFIER_SQL2, SELECT_MODIFIER_PRIORITY2, JOIN_TYPE_SQL2, CompiledQuery2, DEFAULT_MIGRATION_TABLE2 = "kysely_migration", DEFAULT_MIGRATION_LOCK_TABLE2 = "kysely_migration_lock", DEFAULT_ALLOW_UNORDERED_MIGRATIONS2 = false, MIGRATION_LOCK_ID2 = "migration_lock", NO_MIGRATIONS2, MigrationResultSetError2, LOCK_ID3, DocumentDeletedError, DocumentNotFoundError, EventBusAggregateError, ReactorEventTypes, QueueEventTypes, ModuleNotFoundError, DuplicateModuleError, STRICT_ORDER_ACTION_TYPES, DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive", ProcessorManager, exports_001_create_operation_table, exports_002_create_keyframe_table, exports_003_create_document_table, exports_004_create_document_relationship_table, exports_005_create_indexer_state_table, exports_006_create_document_snapshot_table, exports_007_create_slug_mapping_table, exports_008_create_view_state_table, exports_009_create_operation_index_tables, exports_010_create_sync_tables, exports_011_add_cursor_type_column, exports_012_add_source_remote_column, exports_013_create_sync_dead_letters_table, exports_014_create_processor_cursor_table, REACTOR_SCHEMA = "reactor", migrations, ChannelScheme, SyncOperationStatus, ChannelErrorSource, SyncEventTypes, MailboxAggregateError, ChannelError, SyncOperationAggregateError, syncOpCounter = 0, getLatestAppliedOrdinal = (syncOps) => {
+}, byteToHex, rnds8, randomUUID, PropagationMode, RelationshipChangeType, JobStatus, DocumentChangeType, AlterTableNode2, IdentifierNode2, CreateIndexNode2, CreateSchemaNode2, ON_COMMIT_ACTIONS2, CreateTableNode2, SchemableIdentifierNode2, DropIndexNode2, DropSchemaNode2, DropTableNode2, AliasNode2, TableNode2, SelectModifierNode2, AndNode2, OrNode2, OnNode2, JoinNode2, BinaryOperationNode2, COMPARISON_OPERATORS2, ARITHMETIC_OPERATORS2, JSON_OPERATORS2, BINARY_OPERATORS2, UNARY_FILTER_OPERATORS2, UNARY_OPERATORS2, OPERATORS2, OperatorNode2, ColumnNode2, SelectAllNode2, ReferenceNode2, OrderByItemNode2, RawNode2, CollateNode2, LOGGED_MESSAGES2, JSONReferenceNode2, JSONOperatorChainNode2, JSONPathNode2, PrimitiveValueListNode2, ValueListNode2, ValueNode2, ParensNode2, OrderByNode2, PartitionByNode2, OverNode2, FromNode2, GroupByNode2, HavingNode2, InsertQueryNode2, ListNode2, UpdateQueryNode2, UsingNode2, DeleteQueryNode2, WhereNode2, ReturningNode2, ExplainNode2, WhenNode2, MergeQueryNode2, OutputNode2, QueryNode2, SelectQueryNode2, PartitionByItemNode2, SelectionNode2, ValuesNode2, DefaultInsertValueNode2, ColumnUpdateNode2, OnDuplicateKeyNode2, NoResultError2, OnConflictNode2, TopNode2, OrActionNode2, LimitNode2, CommonTableExpressionNameNode2, CommonTableExpressionNode2, WithNode2, CHARS2, ROOT_OPERATION_NODES2, SCHEMALESS_FUNCTIONS2, WithSchemaTransformer2, MatchedNode2, NO_PLUGINS2, NoopQueryExecutor2, NOOP_QUERY_EXECUTOR2, OffsetNode2, GroupByItemNode2, SetOperationNode2, FetchNode2, AggregateFunctionNode2, FunctionNode2, UnaryOperationNode2, CaseNode2, JSONPathLegNode2, TraversedJSONPathBuilder2, TupleNode2, SIMPLE_COLUMN_DATA_TYPES2, COLUMN_DATA_TYPE_REGEX2, DataTypeNode2, CastNode2, AddColumnNode2, ColumnDefinitionNode2, DropColumnNode2, RenameColumnNode2, CheckConstraintNode2, ON_MODIFY_FOREIGN_ACTIONS2, ReferencesNode2, GeneratedNode2, DefaultValueNode2, ModifyColumnNode2, ForeignKeyConstraintNode2, AddConstraintNode2, UniqueConstraintNode2, DropConstraintNode2, AlterColumnNode2, PrimaryKeyConstraintNode2, AddIndexNode2, RenameConstraintNode2, ImmediateValueTransformer2, CreateViewNode2, DropViewNode2, CreateTypeNode2, DropTypeNode2, RefreshMaterializedViewNode2, DefaultQueryExecutor2, ignoreError2 = () => {}, TRANSACTION_ACCESS_MODES2, TRANSACTION_ISOLATION_LEVELS2, logLevels2, LOG_LEVELS4, Kysely2, Transaction2, ControlledTransaction2, sql3, LIT_WRAP_REGEX2, DefaultQueryCompiler2, SELECT_MODIFIER_SQL2, SELECT_MODIFIER_PRIORITY2, JOIN_TYPE_SQL2, CompiledQuery2, DEFAULT_MIGRATION_TABLE2 = "kysely_migration", DEFAULT_MIGRATION_LOCK_TABLE2 = "kysely_migration_lock", DEFAULT_ALLOW_UNORDERED_MIGRATIONS2 = false, MIGRATION_LOCK_ID2 = "migration_lock", NO_MIGRATIONS2, MigrationResultSetError2, LOCK_ID3, DocumentDeletedError, DocumentNotFoundError, EventBusAggregateError, ReactorEventTypes, QueueEventTypes, ModuleNotFoundError, DuplicateModuleError, STRICT_ORDER_ACTION_TYPES, DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive", ProcessorManager, exports_001_create_operation_table, exports_002_create_keyframe_table, exports_003_create_document_table, exports_004_create_document_relationship_table, exports_005_create_indexer_state_table, exports_006_create_document_snapshot_table, exports_007_create_slug_mapping_table, exports_008_create_view_state_table, exports_009_create_operation_index_tables, exports_010_create_sync_tables, exports_011_add_cursor_type_column, exports_012_add_source_remote_column, exports_013_create_sync_dead_letters_table, exports_014_create_processor_cursor_table, REACTOR_SCHEMA = "reactor", migrations, ChannelScheme, SyncOperationStatus, ChannelErrorSource, SyncEventTypes, MailboxAggregateError, GraphQLRequestError, ChannelError, SyncOperationAggregateError, syncOpCounter = 0, getLatestAppliedOrdinal = (syncOps) => {
   let maxOrdinal = 0;
   for (const syncOp of syncOps) {
     if (syncOp.status === 2) {
@@ -338255,6 +338318,16 @@ var init_src6 = __esm(() => {
       this.errors = errors5;
     }
   };
+  GraphQLRequestError = class GraphQLRequestError extends Error {
+    statusCode;
+    category;
+    constructor(message, category, statusCode) {
+      super(message);
+      this.name = "GraphQLRequestError";
+      this.category = category;
+      this.statusCode = statusCode;
+    }
+  };
   ChannelError = class ChannelError extends Error {
     source;
     error;