@harperfast/harper 5.2.13 → 5.3.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/mcpTools.ts +1 -1
- package/agent/session.ts +25 -14
- package/bin/cliOperations.ts +58 -9
- package/bin/copyDb.ts +282 -59
- package/bin/deploySetup.ts +16 -5
- package/bin/harper.ts +1 -1
- package/bin/help.ts +4 -1
- package/bin/lite.ts +4 -1
- package/bin/restart.ts +120 -4
- package/bin/run.ts +6 -11
- package/bin/upgrade.js +7 -3
- package/bin/workloadIdentity.ts +119 -0
- package/components/Application.ts +3436 -242
- package/components/ApplicationScope.ts +8 -0
- package/components/EntryHandler.ts +59 -39
- package/components/OptionsWatcher.ts +440 -98
- package/components/RuntimeModuleTracker.ts +38 -7
- package/components/Scope.ts +56 -15
- package/components/awaitRestart.ts +84 -0
- package/components/componentLoader.ts +381 -32
- package/components/componentPreparationLock.ts +16 -5
- package/components/deploymentOperations.ts +4 -1
- package/components/deploymentRecorder.ts +9 -2
- package/components/mcp/adapters/harperHttp.ts +4 -0
- package/components/mcp/listChanged.ts +4 -0
- package/components/mcp/toolRegistry.ts +2 -0
- package/components/mcp/tools/operations.ts +9 -0
- package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
- package/components/operations.js +537 -113
- package/components/operationsValidation.js +98 -3
- package/components/packageComponent.ts +25 -1
- package/components/requestRestart.ts +11 -0
- package/components/status/ComponentStatusRegistry.ts +59 -0
- package/config/RootConfigWatcher.ts +240 -40
- package/config/configReadRetry.ts +62 -0
- package/config/configUtils.ts +357 -48
- package/config/harperConfigEnvVars.ts +170 -27
- package/config/parseConfigFile.ts +34 -0
- package/config/readConfigFileSync.ts +44 -0
- package/config/watcherArming.ts +59 -0
- package/config-root.schema.json +33 -0
- package/dataLayer/blobBackup.ts +160 -50
- package/dataLayer/delete.ts +6 -1
- package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
- package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dataLayer/hdbInfoController.ts +34 -1
- package/dataLayer/insert.ts +44 -1
- package/dataLayer/rocksdbBackup.ts +53 -10
- package/dataLayer/schema.ts +11 -1
- package/dataLayer/schemaDescribe.ts +8 -1
- package/dist/agent/mcpTools.js +1 -1
- package/dist/agent/mcpTools.js.map +1 -1
- package/dist/agent/session.d.ts +22 -0
- package/dist/agent/session.js +26 -15
- package/dist/agent/session.js.map +1 -1
- package/dist/bin/cliOperations.js +61 -9
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/copyDb.d.ts +12 -1
- package/dist/bin/copyDb.js +248 -60
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deploySetup.d.ts +2 -0
- package/dist/bin/deploySetup.js +11 -3
- package/dist/bin/deploySetup.js.map +1 -1
- package/dist/bin/harper.js +1 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/bin/help.js +4 -1
- package/dist/bin/help.js.map +1 -1
- package/dist/bin/lite.js +4 -1
- package/dist/bin/lite.js.map +1 -1
- package/dist/bin/restart.js +91 -6
- package/dist/bin/restart.js.map +1 -1
- package/dist/bin/run.js +4 -10
- package/dist/bin/run.js.map +1 -1
- package/dist/bin/upgrade.js +4 -3
- package/dist/bin/upgrade.js.map +1 -1
- package/dist/bin/workloadIdentity.d.ts +18 -0
- package/dist/bin/workloadIdentity.js +100 -0
- package/dist/bin/workloadIdentity.js.map +1 -0
- package/dist/components/Application.d.ts +235 -17
- package/dist/components/Application.js +3038 -238
- package/dist/components/Application.js.map +1 -1
- package/dist/components/ApplicationScope.d.ts +8 -0
- package/dist/components/ApplicationScope.js +7 -0
- package/dist/components/ApplicationScope.js.map +1 -1
- package/dist/components/EntryHandler.js +26 -10
- package/dist/components/EntryHandler.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +4 -0
- package/dist/components/OptionsWatcher.js +440 -99
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/RuntimeModuleTracker.js +40 -6
- package/dist/components/RuntimeModuleTracker.js.map +1 -1
- package/dist/components/Scope.js +52 -13
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/awaitRestart.d.ts +33 -0
- package/dist/components/awaitRestart.js +61 -0
- package/dist/components/awaitRestart.js.map +1 -0
- package/dist/components/componentLoader.d.ts +38 -1
- package/dist/components/componentLoader.js +313 -24
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/componentPreparationLock.d.ts +5 -0
- package/dist/components/componentPreparationLock.js +14 -6
- package/dist/components/componentPreparationLock.js.map +1 -1
- package/dist/components/deploymentOperations.js +4 -1
- package/dist/components/deploymentOperations.js.map +1 -1
- package/dist/components/deploymentRecorder.d.ts +4 -2
- package/dist/components/deploymentRecorder.js +1 -0
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/adapters/harperHttp.js +4 -0
- package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
- package/dist/components/mcp/listChanged.js +5 -0
- package/dist/components/mcp/listChanged.js.map +1 -1
- package/dist/components/mcp/toolRegistry.d.ts +1 -0
- package/dist/components/mcp/toolRegistry.js.map +1 -1
- package/dist/components/mcp/tools/operations.d.ts +5 -0
- package/dist/components/mcp/tools/operations.js +9 -0
- package/dist/components/mcp/tools/operations.js.map +1 -1
- package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
- package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
- package/dist/components/operations.d.ts +28 -0
- package/dist/components/operations.js +476 -113
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +97 -3
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/packageComponent.js +24 -0
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/components/requestRestart.d.ts +1 -0
- package/dist/components/requestRestart.js +7 -0
- package/dist/components/requestRestart.js.map +1 -1
- package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
- package/dist/components/status/ComponentStatusRegistry.js +63 -0
- package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +9 -1
- package/dist/config/RootConfigWatcher.js +228 -37
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configReadRetry.d.ts +8 -0
- package/dist/config/configReadRetry.js +62 -0
- package/dist/config/configReadRetry.js.map +1 -0
- package/dist/config/configUtils.d.ts +11 -2
- package/dist/config/configUtils.js +304 -47
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/harperConfigEnvVars.d.ts +16 -0
- package/dist/config/harperConfigEnvVars.js +162 -25
- package/dist/config/harperConfigEnvVars.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +4 -0
- package/dist/config/parseConfigFile.js +35 -0
- package/dist/config/parseConfigFile.js.map +1 -0
- package/dist/config/readConfigFileSync.d.ts +1 -0
- package/dist/config/readConfigFileSync.js +47 -0
- package/dist/config/readConfigFileSync.js.map +1 -0
- package/dist/config/watcherArming.d.ts +15 -0
- package/dist/config/watcherArming.js +59 -0
- package/dist/config/watcherArming.js.map +1 -0
- package/dist/dataLayer/blobBackup.d.ts +49 -20
- package/dist/dataLayer/blobBackup.js +139 -50
- package/dist/dataLayer/blobBackup.js.map +1 -1
- package/dist/dataLayer/delete.js +1 -1
- package/dist/dataLayer/delete.js.map +1 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
- package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
- package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
- package/dist/dataLayer/hdbInfoController.d.ts +10 -0
- package/dist/dataLayer/hdbInfoController.js +30 -1
- package/dist/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/dataLayer/insert.d.ts +9 -1
- package/dist/dataLayer/insert.js +30 -0
- package/dist/dataLayer/insert.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
- package/dist/dataLayer/rocksdbBackup.js +45 -8
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/dataLayer/schema.js +8 -0
- package/dist/dataLayer/schema.js.map +1 -1
- package/dist/dataLayer/schemaDescribe.js +8 -1
- package/dist/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +55 -0
- package/dist/resources/DatabaseTransaction.d.ts +77 -0
- package/dist/resources/DatabaseTransaction.js +586 -54
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +2 -1
- package/dist/resources/LMDBTransaction.js +43 -6
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
- package/dist/resources/PrimaryRocksDatabase.js +49 -5
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +20 -0
- package/dist/resources/RecordEncoder.js +110 -9
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/RequestTarget.d.ts +2 -0
- package/dist/resources/RequestTarget.js.map +1 -1
- package/dist/resources/Resource.js +117 -24
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +28 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +6 -1
- package/dist/resources/RocksIndexStore.js +24 -9
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
- package/dist/resources/RocksTransactionLogStore.js +160 -49
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +165 -10
- package/dist/resources/Table.js +1805 -303
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +10 -3
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +181 -2
- package/dist/resources/auditStore.js +640 -29
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/blob.d.ts +129 -9
- package/dist/resources/blob.js +992 -114
- package/dist/resources/blob.js.map +1 -1
- package/dist/resources/branchDatabase.d.ts +48 -0
- package/dist/resources/branchDatabase.js +892 -0
- package/dist/resources/branchDatabase.js.map +1 -0
- package/dist/resources/crdt.js +50 -12
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/dataLoader.js +3 -4
- package/dist/resources/dataLoader.js.map +1 -1
- package/dist/resources/databases.d.ts +157 -14
- package/dist/resources/databases.js +1462 -325
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/defineTable.d.ts +10 -2
- package/dist/resources/defineTable.js +9 -1
- package/dist/resources/defineTable.js.map +1 -1
- package/dist/resources/derivedIndexRegistry.d.ts +5 -0
- package/dist/resources/derivedIndexRegistry.js +68 -0
- package/dist/resources/derivedIndexRegistry.js.map +1 -0
- package/dist/resources/derivedIndexRuntime.d.ts +215 -0
- package/dist/resources/derivedIndexRuntime.js +2027 -0
- package/dist/resources/derivedIndexRuntime.js.map +1 -0
- package/dist/resources/graphql.d.ts +1 -1
- package/dist/resources/graphql.js +52 -16
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
- package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
- package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
- package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
- package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
- package/dist/resources/longLivedTransactions.d.ts +71 -0
- package/dist/resources/longLivedTransactions.js +358 -0
- package/dist/resources/longLivedTransactions.js.map +1 -0
- package/dist/resources/models/backendRegistry.d.ts +26 -0
- package/dist/resources/models/backendRegistry.js +60 -2
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +33 -1
- package/dist/resources/models/bootstrap.js +416 -31
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/nodeIdMapping.d.ts +5 -0
- package/dist/resources/nodeIdMapping.js +49 -0
- package/dist/resources/nodeIdMapping.js.map +1 -1
- package/dist/resources/recordLock.d.ts +123 -0
- package/dist/resources/recordLock.js +315 -0
- package/dist/resources/recordLock.js.map +1 -0
- package/dist/resources/recordLockCoordinator.d.ts +557 -0
- package/dist/resources/recordLockCoordinator.js +2565 -0
- package/dist/resources/recordLockCoordinator.js.map +1 -0
- package/dist/resources/replayLogs.d.ts +14 -1
- package/dist/resources/replayLogs.js +157 -24
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replayLogsGuards.d.ts +94 -7
- package/dist/resources/replayLogsGuards.js +111 -7
- package/dist/resources/replayLogsGuards.js.map +1 -1
- package/dist/resources/replicatedApplyFailure.d.ts +16 -0
- package/dist/resources/replicatedApplyFailure.js +63 -0
- package/dist/resources/replicatedApplyFailure.js.map +1 -0
- package/dist/resources/scheduler/scheduler.js +3 -3
- package/dist/resources/scheduler/scheduler.js.map +1 -1
- package/dist/resources/search.d.ts +10 -4
- package/dist/resources/search.js +283 -37
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/tracked.d.ts +5 -1
- package/dist/resources/tracked.js +74 -23
- package/dist/resources/tracked.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +1 -1
- package/dist/resources/transactionBroadcast.js +2 -2
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +61 -30
- package/dist/security/auth.js.map +1 -1
- package/dist/security/authn/oidc/claims.d.ts +22 -0
- package/dist/security/authn/oidc/claims.js +71 -0
- package/dist/security/authn/oidc/claims.js.map +1 -0
- package/dist/security/authn/oidc/identityToken.d.ts +27 -0
- package/dist/security/authn/oidc/identityToken.js +111 -0
- package/dist/security/authn/oidc/identityToken.js.map +1 -0
- package/dist/security/authn/oidc/jwks.d.ts +25 -0
- package/dist/security/authn/oidc/jwks.js +261 -0
- package/dist/security/authn/oidc/jwks.js.map +1 -0
- package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
- package/dist/security/authn/oidc/providers/generic.js +34 -0
- package/dist/security/authn/oidc/providers/generic.js.map +1 -0
- package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
- package/dist/security/authn/oidc/providers/githubActions.js +129 -0
- package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
- package/dist/security/authn/oidc/providers/index.d.ts +37 -0
- package/dist/security/authn/oidc/providers/index.js +24 -0
- package/dist/security/authn/oidc/providers/index.js.map +1 -0
- package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
- package/dist/security/authn/oidc/tokenExchange.js +306 -0
- package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
- package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
- package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
- package/dist/security/authn/oidc/types.d.ts +38 -0
- package/dist/security/authn/oidc/types.js +6 -0
- package/dist/security/authn/oidc/types.js.map +1 -0
- package/dist/security/certificateVerification/index.js +40 -11
- package/dist/security/certificateVerification/index.js.map +1 -1
- package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
- package/dist/security/certificateVerification/trustedIssuers.js +79 -0
- package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
- package/dist/security/certificateVerification/types.d.ts +1 -0
- package/dist/security/credentialProvenance.d.ts +35 -0
- package/dist/security/credentialProvenance.js +51 -0
- package/dist/security/credentialProvenance.js.map +1 -0
- package/dist/security/credentialRejection.d.ts +4 -0
- package/dist/security/credentialRejection.js +24 -0
- package/dist/security/credentialRejection.js.map +1 -0
- package/dist/security/deferredAuthentication.d.ts +36 -0
- package/dist/security/deferredAuthentication.js +70 -0
- package/dist/security/deferredAuthentication.js.map +1 -0
- package/dist/security/impersonation.d.ts +21 -0
- package/dist/security/impersonation.js +108 -9
- package/dist/security/impersonation.js.map +1 -1
- package/dist/security/jsLoader.d.ts +6 -0
- package/dist/security/jsLoader.js +77 -15
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/security/keys.js +301 -71
- package/dist/security/keys.js.map +1 -1
- package/dist/security/operationScope.d.ts +21 -0
- package/dist/security/operationScope.js +36 -0
- package/dist/security/operationScope.js.map +1 -0
- package/dist/security/permissionsTranslator.js +21 -0
- package/dist/security/permissionsTranslator.js.map +1 -1
- package/dist/security/tokenAuthentication.d.ts +19 -1
- package/dist/security/tokenAuthentication.js +191 -10
- package/dist/security/tokenAuthentication.js.map +1 -1
- package/dist/security/user.js +4 -3
- package/dist/security/user.js.map +1 -1
- package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
- package/dist/server/DurableSubscriptionsSession.js +67 -10
- package/dist/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/server/REST.js +106 -2
- package/dist/server/REST.js.map +1 -1
- package/dist/server/graphqlQuerying.js +4 -0
- package/dist/server/graphqlQuerying.js.map +1 -1
- package/dist/server/http.d.ts +16 -1
- package/dist/server/http.js +122 -17
- package/dist/server/http.js.map +1 -1
- package/dist/server/itc/serverHandlers.js +8 -1
- package/dist/server/itc/serverHandlers.js.map +1 -1
- package/dist/server/jobs/jobProcess.js +6 -2
- package/dist/server/jobs/jobProcess.js.map +1 -1
- package/dist/server/jobs/jobs.js +4 -1
- package/dist/server/jobs/jobs.js.map +1 -1
- package/dist/server/liveSubscriptionAuth.d.ts +26 -4
- package/dist/server/liveSubscriptionAuth.js +105 -39
- package/dist/server/liveSubscriptionAuth.js.map +1 -1
- package/dist/server/loadRootComponents.js +49 -10
- package/dist/server/loadRootComponents.js.map +1 -1
- package/dist/server/mqtt.d.ts +2 -0
- package/dist/server/mqtt.js +165 -30
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/nodeName.d.ts +2 -0
- package/dist/server/nodeName.js +107 -23
- package/dist/server/nodeName.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +27 -0
- package/dist/server/serverHelpers/Headers.js +145 -1
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
- package/dist/server/serverHelpers/Request.d.ts +5 -10
- package/dist/server/serverHelpers/Request.js +38 -136
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
- package/dist/server/serverHelpers/contentTypes.js +221 -40
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
- package/dist/server/serverHelpers/deployValidationState.js +9 -19
- package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
- package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
- package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
- package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
- package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
- package/dist/server/serverHelpers/registeredOperations.js +74 -21
- package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
- package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
- package/dist/server/serverHelpers/requestSanitization.js +20 -0
- package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
- package/dist/server/serverHelpers/serverHandlers.js +6 -3
- package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
- package/dist/server/serverHelpers/serverUtilities.js +178 -29
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
- package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
- package/dist/server/serverHelpers/uwsServer.js +19 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/static.js +24 -28
- package/dist/server/static.js.map +1 -1
- package/dist/server/storageReclamation.d.ts +5 -0
- package/dist/server/storageReclamation.js +17 -1
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/isolatedApplications.d.ts +47 -0
- package/dist/server/threads/isolatedApplications.js +171 -0
- package/dist/server/threads/isolatedApplications.js.map +1 -0
- package/dist/server/threads/itc.d.ts +7 -2
- package/dist/server/threads/itc.js +5 -1
- package/dist/server/threads/itc.js.map +1 -1
- package/dist/server/threads/logRotationTransport.d.ts +1 -0
- package/dist/server/threads/logRotationTransport.js +33 -0
- package/dist/server/threads/logRotationTransport.js.map +1 -0
- package/dist/server/threads/manageThreads.d.ts +83 -3
- package/dist/server/threads/manageThreads.js +769 -75
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -0
- package/dist/server/threads/socketRouter.js +277 -31
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadHeapMemory.d.ts +2 -0
- package/dist/server/threads/threadHeapMemory.js +31 -0
- package/dist/server/threads/threadHeapMemory.js.map +1 -0
- package/dist/server/threads/threadServer.js +76 -22
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/sqlEngine/config.d.ts +1 -3
- package/dist/sqlEngine/config.js +19 -16
- package/dist/sqlEngine/config.js.map +1 -1
- package/dist/sqlTranslator/index.d.ts +1 -1
- package/dist/sqlTranslator/index.js +30 -7
- package/dist/sqlTranslator/index.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.d.ts +7 -0
- package/dist/upgrade/directives/5-3-0.js +148 -0
- package/dist/upgrade/directives/5-3-0.js.map +1 -0
- package/dist/upgrade/directives/directivesController.js +2 -1
- package/dist/upgrade/directives/directivesController.js.map +1 -1
- package/dist/utility/OperationFunctionCaller.js +2 -1
- package/dist/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/utility/common_utils.d.ts +16 -0
- package/dist/utility/common_utils.js +32 -6
- package/dist/utility/common_utils.js.map +1 -1
- package/dist/utility/componentNames.d.ts +8 -0
- package/dist/utility/componentNames.js +12 -1
- package/dist/utility/componentNames.js.map +1 -1
- package/dist/utility/environment/environmentManager.js +3 -6
- package/dist/utility/environment/environmentManager.js.map +1 -1
- package/dist/utility/environment/systemInformation.d.ts +1 -0
- package/dist/utility/environment/systemInformation.js +1 -0
- package/dist/utility/environment/systemInformation.js.map +1 -1
- package/dist/utility/errors/commonErrors.d.ts +2 -0
- package/dist/utility/errors/commonErrors.js +2 -0
- package/dist/utility/errors/commonErrors.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +33 -0
- package/dist/utility/errors/hdbError.js +58 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +18 -0
- package/dist/utility/hdbTerms.d.ts +18 -0
- package/dist/utility/hdbTerms.js +20 -2
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.d.ts +2 -0
- package/dist/utility/logging/harper_logger.js +286 -30
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
- package/dist/utility/logging/logGenerationCoordinator.js +184 -0
- package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
- package/dist/utility/logging/logRotation.d.ts +46 -0
- package/dist/utility/logging/logRotation.js +365 -0
- package/dist/utility/logging/logRotation.js.map +1 -0
- package/dist/utility/logging/logRotator.d.ts +1 -1
- package/dist/utility/logging/logRotator.js +192 -85
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/nodeIdentity.d.ts +9 -0
- package/dist/utility/nodeIdentity.js +58 -0
- package/dist/utility/nodeIdentity.js.map +1 -0
- package/dist/utility/npmUtilities.js +11 -7
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/operationPermissions.d.ts +3 -1
- package/dist/utility/operationPermissions.js +16 -1
- package/dist/utility/operationPermissions.js.map +1 -1
- package/dist/utility/operation_authorization.d.ts +10 -7
- package/dist/utility/operation_authorization.js +212 -41
- package/dist/utility/operation_authorization.js.map +1 -1
- package/dist/utility/watchPath.d.ts +29 -0
- package/dist/utility/watchPath.js +68 -0
- package/dist/utility/watchPath.js.map +1 -0
- package/dist/utility/watcherFallback.d.ts +41 -0
- package/dist/utility/watcherFallback.js +153 -0
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.d.ts +12 -0
- package/dist/validation/configValidator.js +205 -75
- package/dist/validation/configValidator.js.map +1 -1
- package/dist/validation/installValidator.js +12 -0
- package/dist/validation/installValidator.js.map +1 -1
- package/dist/validation/validationWrapper.d.ts +11 -0
- package/dist/validation/validationWrapper.js +16 -3
- package/dist/validation/validationWrapper.js.map +1 -1
- package/index.ts +8 -0
- package/json/systemSchema.json +55 -0
- package/npm-shrinkwrap.json +286 -194
- package/package.json +15 -7
- package/resources/DESIGN.md +219 -52
- package/resources/DatabaseTransaction.ts +657 -52
- package/resources/LMDBTransaction.ts +46 -6
- package/resources/PrimaryRocksDatabase.ts +46 -7
- package/resources/RecordEncoder.ts +124 -9
- package/resources/RequestTarget.ts +2 -0
- package/resources/Resource.ts +114 -22
- package/resources/ResourceInterface.ts +31 -0
- package/resources/RocksIndexStore.ts +30 -9
- package/resources/RocksTransactionLogStore.ts +188 -51
- package/resources/Table.ts +1960 -301
- package/resources/analytics/write.ts +10 -3
- package/resources/auditStore.ts +647 -32
- package/resources/blob.ts +1029 -110
- package/resources/branchDatabase.ts +941 -0
- package/resources/crdt.ts +76 -12
- package/resources/dataLoader.ts +3 -4
- package/resources/databases.ts +1584 -320
- package/resources/defineTable.ts +18 -2
- package/resources/derivedIndexRegistry.ts +56 -0
- package/resources/derivedIndexRuntime.ts +2292 -0
- package/resources/graphql.ts +72 -17
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
- package/resources/indexes/hnswDerivedIndex.ts +531 -0
- package/resources/indexes/hnswPlaneBinding.ts +174 -0
- package/resources/longLivedTransactions.ts +360 -0
- package/resources/models/backendRegistry.ts +84 -2
- package/resources/models/bootstrap.ts +473 -28
- package/resources/nodeIdMapping.ts +50 -0
- package/resources/recordLock.ts +419 -0
- package/resources/recordLockCoordinator.ts +3043 -0
- package/resources/replayLogs.ts +152 -26
- package/resources/replayLogsGuards.ts +171 -8
- package/resources/replicatedApplyFailure.ts +77 -0
- package/resources/scheduler/scheduler.ts +4 -4
- package/resources/search.ts +288 -46
- package/resources/tracked.ts +73 -22
- package/resources/transactionBroadcast.ts +3 -3
- package/security/auth.ts +68 -29
- package/security/authn/oidc/claims.ts +72 -0
- package/security/authn/oidc/identityToken.ts +129 -0
- package/security/authn/oidc/jwks.ts +260 -0
- package/security/authn/oidc/providers/generic.ts +40 -0
- package/security/authn/oidc/providers/githubActions.ts +137 -0
- package/security/authn/oidc/providers/index.ts +52 -0
- package/security/authn/oidc/tokenExchange.ts +300 -0
- package/security/authn/oidc/trustPolicyOperations.ts +343 -0
- package/security/authn/oidc/types.ts +41 -0
- package/security/certificateVerification/index.ts +54 -13
- package/security/certificateVerification/trustedIssuers.ts +76 -0
- package/security/certificateVerification/types.ts +1 -0
- package/security/credentialProvenance.ts +47 -0
- package/security/credentialRejection.ts +22 -0
- package/security/deferredAuthentication.ts +71 -0
- package/security/impersonation.ts +117 -12
- package/security/jsLoader.ts +83 -18
- package/security/keys.ts +298 -72
- package/security/operationScope.ts +33 -0
- package/security/permissionsTranslator.js +23 -0
- package/security/tokenAuthentication.ts +233 -12
- package/security/user.ts +4 -3
- package/server/DESIGN.md +194 -16
- package/server/DurableSubscriptionsSession.ts +71 -11
- package/server/REST.ts +115 -4
- package/server/graphqlQuerying.ts +4 -0
- package/server/http.ts +133 -20
- package/server/itc/serverHandlers.js +8 -1
- package/server/jobs/jobProcess.ts +8 -2
- package/server/jobs/jobs.ts +4 -1
- package/server/liveSubscriptionAuth.ts +129 -46
- package/server/loadRootComponents.js +50 -8
- package/server/mqtt.ts +179 -38
- package/server/nodeName.ts +103 -21
- package/server/serverHelpers/Headers.ts +135 -0
- package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
- package/server/serverHelpers/Request.ts +33 -131
- package/server/serverHelpers/contentTypes.ts +217 -36
- package/server/serverHelpers/deployValidationState.ts +24 -13
- package/server/serverHelpers/operationAuthorizationState.ts +34 -3
- package/server/serverHelpers/registeredOperations.ts +79 -22
- package/server/serverHelpers/requestSanitization.ts +15 -0
- package/server/serverHelpers/serverHandlers.js +6 -3
- package/server/serverHelpers/serverUtilities.ts +232 -40
- package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
- package/server/serverHelpers/uwsServer.ts +17 -2
- package/server/static.ts +23 -29
- package/server/storageReclamation.ts +15 -2
- package/server/threads/isolatedApplications.ts +157 -0
- package/server/threads/itc.js +11 -1
- package/server/threads/logRotationTransport.ts +40 -0
- package/server/threads/manageThreads.js +771 -66
- package/server/threads/socketRouter.ts +291 -30
- package/server/threads/threadHeapMemory.ts +26 -0
- package/server/threads/threadServer.js +73 -22
- package/sqlTranslator/index.ts +31 -8
- package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
- package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
- package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
- package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
- package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
- package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
- package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
- package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
- package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
- package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
- package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
- package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
- package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
- package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
- package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
- package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
- package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
- package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
- package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
- package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
- package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
- package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
- package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
- package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
- package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
- package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
- package/studio/web/index.html +1 -1
- package/upgrade/directives/5-3-0.ts +132 -0
- package/upgrade/directives/directivesController.ts +2 -1
- package/utility/OperationFunctionCaller.ts +2 -1
- package/utility/common_utils.ts +30 -5
- package/utility/componentNames.ts +12 -0
- package/utility/environment/environmentManager.ts +3 -7
- package/utility/environment/systemInformation.ts +7 -0
- package/utility/errors/commonErrors.ts +4 -0
- package/utility/errors/hdbError.ts +57 -0
- package/utility/hdbTerms.ts +19 -0
- package/utility/logging/harper_logger.ts +278 -26
- package/utility/logging/logGenerationCoordinator.ts +196 -0
- package/utility/logging/logRotation.ts +367 -0
- package/utility/logging/logRotator.ts +213 -81
- package/utility/nodeIdentity.ts +45 -0
- package/utility/npmUtilities.ts +12 -8
- package/utility/operationPermissions.ts +18 -1
- package/utility/operation_authorization.ts +231 -42
- package/utility/watchPath.ts +63 -0
- package/utility/watcherFallback.ts +148 -0
- package/validation/configValidator.ts +215 -75
- package/validation/installValidator.ts +15 -0
- package/validation/validationWrapper.ts +18 -4
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { type Logger } from '../utility/logging/logger.ts';
|
|
2
2
|
import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.ts';
|
|
3
3
|
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
|
|
4
|
+
import { ClientError } from '../utility/errors/hdbError.ts';
|
|
4
5
|
import logger, { errorForLog } from '../utility/logging/harper_logger.ts';
|
|
5
6
|
import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts';
|
|
6
|
-
import { withComponentPreparationLock } from './componentPreparationLock.ts';
|
|
7
|
-
import {
|
|
7
|
+
import { ComponentPreparationLockTimeoutError, withComponentPreparationLock } from './componentPreparationLock.ts';
|
|
8
|
+
import {
|
|
9
|
+
isThreadRunning,
|
|
10
|
+
isProcessGroupAlive,
|
|
11
|
+
registerProcessGroup,
|
|
12
|
+
unregisterProcessGroup,
|
|
13
|
+
} from '../server/threads/manageThreads.js';
|
|
8
14
|
import type { CredentialReference, ResolvedCredential, ResolvedRegistryCredential } from './secretOperations.ts';
|
|
9
15
|
import {
|
|
10
16
|
GIT_CREDENTIAL_SOCKET_ENV,
|
|
@@ -15,16 +21,22 @@ import {
|
|
|
15
21
|
import { getSecretDecryptor } from '../resources/secretDecryptor.ts';
|
|
16
22
|
import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts';
|
|
17
23
|
|
|
18
|
-
import { basename, dirname, extname, join } from 'node:path';
|
|
24
|
+
import { basename, dirname, extname, isAbsolute, join, relative, sep, win32 } from 'node:path';
|
|
19
25
|
import {
|
|
20
26
|
access,
|
|
27
|
+
chmod,
|
|
21
28
|
constants,
|
|
22
|
-
|
|
29
|
+
lstat,
|
|
23
30
|
mkdir,
|
|
31
|
+
link,
|
|
24
32
|
mkdtemp,
|
|
33
|
+
open,
|
|
25
34
|
readdir,
|
|
26
35
|
readFile,
|
|
36
|
+
readlink,
|
|
37
|
+
realpath,
|
|
27
38
|
rename,
|
|
39
|
+
rmdir,
|
|
28
40
|
rm,
|
|
29
41
|
stat,
|
|
30
42
|
symlink,
|
|
@@ -33,7 +45,7 @@ import {
|
|
|
33
45
|
import { spawn, type ChildProcess } from 'node:child_process';
|
|
34
46
|
import { tmpdir } from 'node:os';
|
|
35
47
|
import { randomUUID } from 'node:crypto';
|
|
36
|
-
import { createReadStream, existsSync } from 'node:fs';
|
|
48
|
+
import { chmodSync, createReadStream, existsSync, lstatSync, renameSync } from 'node:fs';
|
|
37
49
|
import { Readable } from 'node:stream';
|
|
38
50
|
import { pipeline } from 'node:stream/promises';
|
|
39
51
|
import { StringDecoder } from 'node:string_decoder';
|
|
@@ -55,6 +67,27 @@ interface ApplicationConfig {
|
|
|
55
67
|
// Recorded by deploy_component so every (cold) install — reboot, new peer, rollback — re-resolves
|
|
56
68
|
// the credential from the store rather than needing it re-supplied.
|
|
57
69
|
credentials?: CredentialReference[];
|
|
70
|
+
/**
|
|
71
|
+
* Databases this application gets a private fork of (harper#642), declared HERE — on the
|
|
72
|
+
* application's root-config entry, alongside `host` and `urlPath` — because which databases an
|
|
73
|
+
* application forks is a deployment decision, not something the application checks in. Declaring
|
|
74
|
+
* it in the application's own config.yaml is refused rather than ignored.
|
|
75
|
+
*
|
|
76
|
+
* `true` forks every database on the instance except `system` -- a snapshot of what exists when
|
|
77
|
+
* this application loads, not a standing subscription: a database created afterward is not
|
|
78
|
+
* retroactively branched.
|
|
79
|
+
*
|
|
80
|
+
* The fork is durable: it lives at a path derived from the application and database names, is
|
|
81
|
+
* adopted again on restart, and is invisible to other applications.
|
|
82
|
+
*
|
|
83
|
+
* The application must reach its data through `import { databases } from 'harper'`. The bare
|
|
84
|
+
* `databases`/`tables` globals are shared process-wide by the default `vm-current-context` loader
|
|
85
|
+
* and cannot be scoped, so an application that uses them reads and writes the BASE — silently.
|
|
86
|
+
* Per-application globals are a property of thread-level isolation, not of branching.
|
|
87
|
+
*/
|
|
88
|
+
branchedDatabases?: string[] | true;
|
|
89
|
+
/** Run in a worker thread of its own that loads no other application. */
|
|
90
|
+
isolated?: boolean;
|
|
58
91
|
// an application config can have other arbitrary properties
|
|
59
92
|
[key: string]: unknown;
|
|
60
93
|
}
|
|
@@ -99,6 +132,12 @@ export class InvalidCredentialsPropertyError extends TypeError {
|
|
|
99
132
|
}
|
|
100
133
|
}
|
|
101
134
|
|
|
135
|
+
export class InvalidBranchedDatabasesError extends TypeError {
|
|
136
|
+
constructor(applicationName: string, detail: string) {
|
|
137
|
+
super(`Invalid 'branchedDatabases' for application ${applicationName}: ${detail}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
102
141
|
export class InvalidCredentialEntryError extends TypeError {
|
|
103
142
|
constructor(applicationName: string) {
|
|
104
143
|
super(
|
|
@@ -112,6 +151,18 @@ export function assertApplicationConfig(
|
|
|
112
151
|
applicationName: string,
|
|
113
152
|
applicationConfig: Record<'package', unknown> & Record<string, unknown>
|
|
114
153
|
): asserts applicationConfig is ApplicationConfig {
|
|
154
|
+
// The deploy staging directory holds a candidate tree under the component's own name beside dot-prefixed
|
|
155
|
+
// control files, so a dot-prefixed component name collides with one of them — and an application named
|
|
156
|
+
// `.activation.json` puts its tree on the journal path, where the journal write takes EEXIST as "a retry
|
|
157
|
+
// of this activation" and the swap proceeds with no journal at all. Rejected HERE rather than tolerated
|
|
158
|
+
// downstream: nothing else validates a root-config application key.
|
|
159
|
+
if (!isJoinableComponentName(applicationName)) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
`Invalid application name '${applicationName}': it must be a single path segment and must not begin ` +
|
|
162
|
+
`with a dot, which is reserved for Harper's own deploy control files`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
115
166
|
if (typeof applicationConfig.package !== 'string') {
|
|
116
167
|
throw new InvalidPackageIdentifierError(applicationName, applicationConfig.package);
|
|
117
168
|
}
|
|
@@ -170,6 +221,47 @@ export function assertApplicationConfig(
|
|
|
170
221
|
}
|
|
171
222
|
}
|
|
172
223
|
}
|
|
224
|
+
assertBranchedDatabases(applicationName, applicationConfig.branchedDatabases);
|
|
225
|
+
assertIsolationConfig(applicationName, applicationConfig.isolated);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export function assertIsolationConfig(applicationName: string, isolated: unknown): void {
|
|
229
|
+
if (isolated !== undefined && typeof isolated !== 'boolean') {
|
|
230
|
+
throw new TypeError(
|
|
231
|
+
`Invalid 'isolated' for application ${applicationName}: expected a boolean, got ${typeof isolated}`
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* A branch that cannot be honoured fails the application's load: falling back would hand it the
|
|
238
|
+
* shared database it asked not to have, with no signal that it happened.
|
|
239
|
+
*/
|
|
240
|
+
export function assertBranchedDatabases(applicationName: string, value: unknown): void {
|
|
241
|
+
if (value === undefined || value === true) return;
|
|
242
|
+
if (!Array.isArray(value)) {
|
|
243
|
+
throw new InvalidBranchedDatabasesError(applicationName, `expected an array or true, got ${typeof value}`);
|
|
244
|
+
}
|
|
245
|
+
const seen = new Set<string>();
|
|
246
|
+
for (const name of value) {
|
|
247
|
+
if (typeof name !== 'string' || name === '') {
|
|
248
|
+
throw new InvalidBranchedDatabasesError(applicationName, `expected database names, got ${typeof name}`);
|
|
249
|
+
}
|
|
250
|
+
// A branch is a directory named by this value; a separator or traversal segment would escape
|
|
251
|
+
// the reserved branch root.
|
|
252
|
+
if (name.includes('/') || name.includes('\\') || name === '.' || name === '..') {
|
|
253
|
+
throw new InvalidBranchedDatabasesError(applicationName, `'${name}' is not a usable database name`);
|
|
254
|
+
}
|
|
255
|
+
// `system` carries the instance's own catalog, users and jobs; a private fork of it would give
|
|
256
|
+
// the application a divergent view of the instance rather than of its data.
|
|
257
|
+
if (name === 'system') {
|
|
258
|
+
throw new InvalidBranchedDatabasesError(applicationName, `the 'system' database cannot be branched`);
|
|
259
|
+
}
|
|
260
|
+
if (seen.has(name)) {
|
|
261
|
+
throw new InvalidBranchedDatabasesError(applicationName, `'${name}' is listed more than once`);
|
|
262
|
+
}
|
|
263
|
+
seen.add(name);
|
|
264
|
+
}
|
|
173
265
|
}
|
|
174
266
|
|
|
175
267
|
/**
|
|
@@ -478,16 +570,51 @@ async function runNpmPack(
|
|
|
478
570
|
// during a deploy swap (see extractApplication). The leading dot keeps
|
|
479
571
|
// loadComponentDirectories from loading its contents as components.
|
|
480
572
|
export const ASIDE_STAGING_DIR = '.deploy-aside';
|
|
573
|
+
// Hidden directory under the components root holding per-deployment candidate builds. A candidate is
|
|
574
|
+
// extracted, installed AND validated here, and only then renamed into the live path, so the previous
|
|
575
|
+
// version keeps serving through the slow, failure-prone work. Dot-prefixed so the three scans over the
|
|
576
|
+
// components root (componentLoader, componentEnvPrepass, resolvePreload) skip it.
|
|
577
|
+
export const DEPLOY_STAGING_DIR = '.deploy-staging';
|
|
578
|
+
const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-';
|
|
579
|
+
const RETIRED_ASIDE_PREFIX = '.retired-';
|
|
580
|
+
const PRIOR_ABSENT_RECORD_SUFFIX = '-prior-absent';
|
|
481
581
|
const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000;
|
|
482
582
|
const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000;
|
|
583
|
+
const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000;
|
|
584
|
+
const COMPONENT_RECOVERY_TRY_TIMEOUT_MS = 250;
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Lock terms for the boot-time activation scan. It runs before every component load on every thread, so it
|
|
588
|
+
* probes rather than queues: the default is a two-hour wait that RENEWS while the holder is alive, which
|
|
589
|
+
* would park a respawning worker behind a deploy's `npm install` and load no components at all until it
|
|
590
|
+
* finished. A held lock means a live deploy, and a live deploy settles its own journal.
|
|
591
|
+
*/
|
|
592
|
+
const RECOVERY_LOCK_WAIT = {
|
|
593
|
+
timeoutMs: COMPONENT_RECOVERY_TRY_TIMEOUT_MS,
|
|
594
|
+
renewTimeoutWhileOwnerAlive: false,
|
|
595
|
+
};
|
|
596
|
+
const COMPONENT_RECOVERY_LOCK_PURPOSE = 'component-recovery';
|
|
483
597
|
const MAX_GIT_EXTRACTION_COMMANDS = 4;
|
|
484
598
|
const MAX_INSTALL_COMMANDS = 2;
|
|
599
|
+
const PRODUCTION_DEPENDENCY_FIELDS = ['dependencies', 'optionalDependencies', 'peerDependencies'] as const;
|
|
600
|
+
const INSTALL_LIFECYCLE_SCRIPTS = new Set([
|
|
601
|
+
'preinstall',
|
|
602
|
+
'install',
|
|
603
|
+
'postinstall',
|
|
604
|
+
'prepublish',
|
|
605
|
+
'preprepare',
|
|
606
|
+
'prepare',
|
|
607
|
+
'postprepare',
|
|
608
|
+
'dependencies',
|
|
609
|
+
]);
|
|
485
610
|
|
|
486
611
|
type ExtractionTransaction = {
|
|
487
612
|
commit(): Promise<void>;
|
|
488
613
|
rollback(): Promise<void>;
|
|
489
614
|
};
|
|
490
615
|
|
|
616
|
+
type ExtractionContext = Pick<Application, 'name' | 'dirPath' | 'logger'>;
|
|
617
|
+
|
|
491
618
|
// The credential helper git executes for a private git-reference deploy. It ships alongside this
|
|
492
619
|
// module (both in source and in dist), holds no secret, and is inert without a live session.
|
|
493
620
|
export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.js');
|
|
@@ -508,6 +635,42 @@ type InstalledPackageMetadata = {
|
|
|
508
635
|
hasInstallableDependencies: boolean;
|
|
509
636
|
};
|
|
510
637
|
|
|
638
|
+
function dependencyFieldHasWork(packageJSON: any, field: string): boolean {
|
|
639
|
+
if (!packageJSON || typeof packageJSON !== 'object' || !Object.hasOwn(packageJSON, field)) return false;
|
|
640
|
+
const value = packageJSON[field];
|
|
641
|
+
return !value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).length > 0;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
export function packageHasProductionInstallWork(packageJSON: any): boolean {
|
|
645
|
+
if (packageJSON === undefined) return false;
|
|
646
|
+
if (!packageJSON || typeof packageJSON !== 'object' || Array.isArray(packageJSON)) return true;
|
|
647
|
+
if (PRODUCTION_DEPENDENCY_FIELDS.some((field) => dependencyFieldHasWork(packageJSON, field))) return true;
|
|
648
|
+
if (!Object.hasOwn(packageJSON, 'workspaces')) return false;
|
|
649
|
+
const workspaces = packageJSON.workspaces;
|
|
650
|
+
if (Array.isArray(workspaces)) return workspaces.length > 0;
|
|
651
|
+
if (!workspaces || typeof workspaces !== 'object' || !Object.hasOwn(workspaces, 'packages')) return true;
|
|
652
|
+
return !Array.isArray(workspaces.packages) || workspaces.packages.length > 0;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
function packageHasExplicitNonNpmManager(packageJSON: any): boolean {
|
|
656
|
+
const packageManager = packageJSON?.devEngines?.packageManager;
|
|
657
|
+
return !!packageManager && packageManager.name !== 'npm';
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
export function packageHasAutomaticInstallWork(packageJSON: any): boolean {
|
|
661
|
+
return packageHasProductionInstallWork(packageJSON) || packageHasExplicitNonNpmManager(packageJSON);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
function packageHasAllowedInstallLifecycleWork(packageJSON: any): boolean {
|
|
665
|
+
if (!packageJSON || typeof packageJSON !== 'object' || !Object.hasOwn(packageJSON, 'scripts')) return false;
|
|
666
|
+
const scripts = packageJSON.scripts;
|
|
667
|
+
if (!scripts || typeof scripts !== 'object' || Array.isArray(scripts)) return true;
|
|
668
|
+
return [...INSTALL_LIFECYCLE_SCRIPTS].some((name) => {
|
|
669
|
+
if (!Object.hasOwn(scripts, name)) return false;
|
|
670
|
+
return typeof scripts[name] !== 'string' || scripts[name].trim().length > 0;
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
|
|
511
674
|
export async function readInstalledPackageMetadata(directory: string): Promise<InstalledPackageMetadata> {
|
|
512
675
|
const files = new Map<string, Buffer>();
|
|
513
676
|
let readable = true;
|
|
@@ -538,17 +701,7 @@ export async function readInstalledPackageMetadata(directory: string): Promise<I
|
|
|
538
701
|
files,
|
|
539
702
|
readable,
|
|
540
703
|
hasLockfile: PACKAGE_LOCK_FILES.some((filename) => files.has(filename)),
|
|
541
|
-
hasInstallableDependencies:
|
|
542
|
-
(field) => {
|
|
543
|
-
const dependencies = packageJSON?.[field];
|
|
544
|
-
return (
|
|
545
|
-
typeof dependencies === 'object' &&
|
|
546
|
-
dependencies !== null &&
|
|
547
|
-
!Array.isArray(dependencies) &&
|
|
548
|
-
Object.keys(dependencies).length > 0
|
|
549
|
-
);
|
|
550
|
-
}
|
|
551
|
-
),
|
|
704
|
+
hasInstallableDependencies: packageHasAutomaticInstallWork(packageJSON),
|
|
552
705
|
};
|
|
553
706
|
}
|
|
554
707
|
|
|
@@ -584,30 +737,17 @@ function canonicalizeJSON(value: any): any {
|
|
|
584
737
|
}
|
|
585
738
|
|
|
586
739
|
/**
|
|
587
|
-
*
|
|
588
|
-
*
|
|
589
|
-
*
|
|
590
|
-
*
|
|
591
|
-
* Writes the application to the configured components root directory using the `application.name` and overwrites any existing directory.
|
|
592
|
-
*
|
|
593
|
-
* This method may be called from any Harper thread. Same-component calls are serialized across
|
|
594
|
-
* threads by the preparation lock below.
|
|
740
|
+
* Either a tarball to extract, or — for `file:<directory>` — an instruction to link that directory. The link
|
|
741
|
+
* case is a RESULT rather than an action so the caller decides where it lands: a candidate build must link
|
|
742
|
+
* at the candidate path, or the deploy is published without validation.
|
|
595
743
|
*/
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
): Promise<ExtractionTransaction | undefined> {
|
|
600
|
-
// Can't specify neither
|
|
601
|
-
if (!application.payload && !application.packageIdentifier) {
|
|
602
|
-
throw new Error('Either payload or package must be provided');
|
|
603
|
-
}
|
|
744
|
+
type ResolvedTarball =
|
|
745
|
+
| { kind: 'tarball'; tarball: Readable; tarballPath?: string; shouldDeleteTarball: boolean }
|
|
746
|
+
| { kind: 'link'; sourceDirPath: string };
|
|
604
747
|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
}
|
|
609
|
-
// Resolve the tarball from the input
|
|
610
|
-
let tarballPath: string;
|
|
748
|
+
/** Resolve `payload` or `package` into a tarball stream. Touches neither the live tree nor staging. */
|
|
749
|
+
async function resolveApplicationTarball(application: Application): Promise<ResolvedTarball> {
|
|
750
|
+
let tarballPath: string | undefined;
|
|
611
751
|
let tarball: Readable;
|
|
612
752
|
let shouldDeleteTarball = false;
|
|
613
753
|
|
|
@@ -626,6 +766,8 @@ export async function extractApplication(
|
|
|
626
766
|
} else {
|
|
627
767
|
// Given a package, there are a a couple options
|
|
628
768
|
const parentDirPath = dirname(application.dirPath);
|
|
769
|
+
let packageIdentifierForPack = application.packageIdentifier;
|
|
770
|
+
let packageNeedsPacking = true;
|
|
629
771
|
|
|
630
772
|
// If the package identifier is a file path we need to check if its a tarball or a directory
|
|
631
773
|
if (application.packageIdentifier.startsWith('file:')) {
|
|
@@ -635,22 +777,29 @@ export async function extractApplication(
|
|
|
635
777
|
const stats = await stat(packagePath);
|
|
636
778
|
|
|
637
779
|
if (stats.isDirectory()) {
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
780
|
+
if (!application.packLocalDirectory) {
|
|
781
|
+
// Reported, not performed — the caller decides where the link goes, so a candidate build
|
|
782
|
+
// links at the candidate path rather than onto the live one.
|
|
783
|
+
return { kind: 'link', sourceDirPath: packagePath };
|
|
784
|
+
}
|
|
785
|
+
// Bare absolute Windows directory inputs historically materialize a copy through npm pack;
|
|
786
|
+
// explicit file: and relative directories remain live links.
|
|
787
|
+
packageIdentifierForPack = packagePath;
|
|
788
|
+
application.logger.debug?.('Packaging local component directory instead of linking it on Windows');
|
|
789
|
+
} else {
|
|
790
|
+
if (!stats.isFile()) {
|
|
791
|
+
throw new Error(`File path specified in package identifier is not a file or directory: ${packagePath}`);
|
|
792
|
+
}
|
|
643
793
|
|
|
644
|
-
|
|
645
|
-
|
|
794
|
+
// If its a file, we assume it can be unzipped and extracted.
|
|
795
|
+
// We are using maybe-gunzip to handle both gzipped and non-gzipped tarballs
|
|
796
|
+
// And then we are happy to let the `tar-fs` library handle the extraction.
|
|
797
|
+
// Maybe worth adding some detection or at least some error handling if that step below fails.
|
|
798
|
+
tarballPath = packagePath;
|
|
799
|
+
tarball = createReadStream(tarballPath);
|
|
800
|
+
packageNeedsPacking = false;
|
|
801
|
+
application.logger.debug?.('Using local component archive directly without npm pack');
|
|
646
802
|
}
|
|
647
|
-
|
|
648
|
-
// If its a file, we assume it can be unzipped and extracted.
|
|
649
|
-
// We are using maybe-gunzip to handle both gzipped and non-gzipped tarballs
|
|
650
|
-
// And then we are happy to let the `tar-fs` library handle the extraction.
|
|
651
|
-
// Maybe worth adding some detection or at least some error handling if that step below fails.
|
|
652
|
-
tarballPath = packagePath;
|
|
653
|
-
tarball = createReadStream(tarballPath);
|
|
654
803
|
} catch (err) {
|
|
655
804
|
if (err.code === 'ENOENT') {
|
|
656
805
|
throw new Error(`File path specified in package identifier does not exist: ${packagePath}`);
|
|
@@ -658,7 +807,8 @@ export async function extractApplication(
|
|
|
658
807
|
throw err;
|
|
659
808
|
}
|
|
660
809
|
}
|
|
661
|
-
}
|
|
810
|
+
}
|
|
811
|
+
if (packageNeedsPacking) {
|
|
662
812
|
// `npm pack --json` writes a JSON array describing the packed tarball(s). This is also the
|
|
663
813
|
// spawn that clones a git-reference package, so it is the only one given the git credential
|
|
664
814
|
// environment.
|
|
@@ -680,195 +830,3013 @@ export async function extractApplication(
|
|
|
680
830
|
// packGitReferenceWithoutScripts), which is exactly what Node 22's bundled npm ships. For a
|
|
681
831
|
// recognized git-reference identifier, clone and pack it ourselves with scripts stripped
|
|
682
832
|
// instead, sidestepping that npm code path entirely.
|
|
683
|
-
const gitRef = allowScripts ? null : parseGitReference(
|
|
833
|
+
const gitRef = allowScripts ? null : parseGitReference(packageIdentifierForPack);
|
|
684
834
|
|
|
685
|
-
if (!allowScripts && !gitRef && looksLikeGitReference(
|
|
835
|
+
if (!allowScripts && !gitRef && looksLikeGitReference(packageIdentifierForPack)) {
|
|
686
836
|
// Recognized as git, but a form the reclone-and-strip-scripts path above can't safely
|
|
687
837
|
// handle (a `#path:` committish, or a hosted shorthand other than a plain `owner/repo`) —
|
|
688
838
|
// fail loudly rather than silently falling through to the unreliable `npm pack
|
|
689
839
|
// --ignore-scripts` below.
|
|
690
840
|
throw new Error(
|
|
691
|
-
`Cannot deploy git-reference package '${
|
|
841
|
+
`Cannot deploy git-reference package '${packageIdentifierForPack}' with install scripts disallowed: this identifier's form (e.g. a '#path:' committish, or a hosted shorthand other than a plain 'owner/repo') isn't one this repo's script-suppression handling supports. Set install.allowInstallScripts to true, or use a plain git URL with a branch/tag/commit committish instead.`
|
|
842
|
+
);
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
if (gitRef) {
|
|
846
|
+
tarballPath = await packGitReferenceWithoutScripts(application, gitRef, parentDirPath);
|
|
847
|
+
} else {
|
|
848
|
+
const packArgs = ['pack', '--json', packageIdentifierForPack];
|
|
849
|
+
if (!allowScripts) {
|
|
850
|
+
packArgs.push('--ignore-scripts');
|
|
851
|
+
} else if (application.gitCredentialEnv) {
|
|
852
|
+
application.logger.warn(
|
|
853
|
+
`Deploying ${application.name} from a git reference with install scripts enabled: the repository's ` +
|
|
854
|
+
`prepare/build scripts and its dependencies' install scripts run on this node during the clone and ` +
|
|
855
|
+
`can read the git credential. Unset install_allow_scripts to keep the credential out of their reach.`
|
|
856
|
+
);
|
|
857
|
+
}
|
|
858
|
+
tarballPath = await runNpmPack(application, packArgs, parentDirPath, application.gitCredentialEnv);
|
|
859
|
+
}
|
|
860
|
+
shouldDeleteTarball = true;
|
|
861
|
+
tarball = createReadStream(tarballPath);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
return { kind: 'tarball', tarball, tarballPath, shouldDeleteTarball };
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/**
|
|
869
|
+
* Extract a tarball into `targetDirPath`, flattening the single wrapping directory npm pack produces.
|
|
870
|
+
* `scratchDirPath` must be on the same filesystem as the target: the flatten is done by renaming the
|
|
871
|
+
* wrapper out and back rather than copying, so it stays atomic per entry. Windows moves the children
|
|
872
|
+
* individually because renaming a directory over its own parent's path fails there.
|
|
873
|
+
*/
|
|
874
|
+
async function extractTarballInto(
|
|
875
|
+
tarball: Readable,
|
|
876
|
+
targetDirPath: string,
|
|
877
|
+
scratchDirPath: string
|
|
878
|
+
): Promise<string | undefined> {
|
|
879
|
+
await mkdir(targetDirPath, { recursive: true });
|
|
880
|
+
await pipeline(tarball, gunzip(), extract(targetDirPath));
|
|
881
|
+
|
|
882
|
+
const extracted = await readdir(targetDirPath, { withFileTypes: true });
|
|
883
|
+
if (extracted.length === 1 && extracted[0].isDirectory()) {
|
|
884
|
+
const topLevelDirPath = join(targetDirPath, extracted[0].name);
|
|
885
|
+
if (process.platform === 'win32') {
|
|
886
|
+
for (const childName of await readdir(topLevelDirPath)) {
|
|
887
|
+
await rename(join(topLevelDirPath, childName), join(targetDirPath, childName));
|
|
888
|
+
}
|
|
889
|
+
await rmdir(topLevelDirPath);
|
|
890
|
+
} else {
|
|
891
|
+
const tempDirPath = join(scratchDirPath, `.normalize-${process.pid}-${Date.now()}-${randomUUID()}`);
|
|
892
|
+
await rename(topLevelDirPath, tempDirPath);
|
|
893
|
+
await rmdir(targetDirPath);
|
|
894
|
+
await rename(tempDirPath, targetDirPath);
|
|
895
|
+
return tempDirPath;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
return undefined;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/**
|
|
902
|
+
* Extract an application given payload (content of the application) or package (npm-compatible identifier to the application).
|
|
903
|
+
*
|
|
904
|
+
* Only one of `application.payload` or `application.package` should be specified; otherwise, an error is thrown.
|
|
905
|
+
*
|
|
906
|
+
* Writes the application to the configured components root directory using the `application.name` and overwrites any existing directory.
|
|
907
|
+
*
|
|
908
|
+
* This method may be called from any Harper thread. Same-component calls are serialized across
|
|
909
|
+
* threads by the preparation lock below.
|
|
910
|
+
*/
|
|
911
|
+
export async function extractApplication(
|
|
912
|
+
application: Application,
|
|
913
|
+
deferCommit = false
|
|
914
|
+
): Promise<ExtractionTransaction | undefined> {
|
|
915
|
+
// Can't specify neither
|
|
916
|
+
if (!application.payload && !application.packageIdentifier) {
|
|
917
|
+
throw new Error('Either payload or package must be provided');
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
// Can't specify both
|
|
921
|
+
if (application.payload && application.packageIdentifier) {
|
|
922
|
+
throw new Error('Both payload and package cannot be provided');
|
|
923
|
+
}
|
|
924
|
+
// Resolve the tarball from the input
|
|
925
|
+
const resolved = await resolveApplicationTarball(application);
|
|
926
|
+
if (resolved.kind === 'link') {
|
|
927
|
+
// Unchanged behavior for this path: a `file:` directory is linked in place, no extraction.
|
|
928
|
+
await symlink(resolved.sourceDirPath, application.dirPath, 'dir');
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
const { tarball, tarballPath, shouldDeleteTarball } = resolved;
|
|
932
|
+
// Replace any existing component directory atomically instead of clearing it in
|
|
933
|
+
// place. A previous version's worker can still be running and actively writing
|
|
934
|
+
// into this directory — e.g. a live Next.js app writing into `.next/cache` — and
|
|
935
|
+
// an in-place recursive rm races that writer: rm empties `.next`, then its leaf
|
|
936
|
+
// `rmdir('.next')` fails with ENOTEMPTY because the worker just re-created a cache
|
|
937
|
+
// entry. (`force: true` only suppresses ENOENT; ENOTEMPTY is not retried unless
|
|
938
|
+
// `maxRetries` is set, and a continuously-writing app would outlast retries
|
|
939
|
+
// anyway.) Renaming the old directory aside is atomic and immune to the race: the
|
|
940
|
+
// still-running worker keeps writing into the renamed inode harmlessly until it's
|
|
941
|
+
// replaced on restart. The aside remains the rollback/recovery record until commit
|
|
942
|
+
// marks it retired and cleanup removes it.
|
|
943
|
+
//
|
|
944
|
+
// The aside lives under a hidden, component-scoped staging directory inside the
|
|
945
|
+
// components root: same filesystem as the source so the rename stays atomic, the
|
|
946
|
+
// leading dot keeps loadComponentDirectories from picking it up as a phantom
|
|
947
|
+
// component, and the per-component path means a sibling component never collides
|
|
948
|
+
// with (or sweeps) another's aside.
|
|
949
|
+
const asideStagingDir = extractionStagingDirectory(application.dirPath);
|
|
950
|
+
const transactionPaths = new Set<string>();
|
|
951
|
+
let asidePath: string | undefined;
|
|
952
|
+
let recoveryRecordPath: string;
|
|
953
|
+
try {
|
|
954
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
955
|
+
await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir);
|
|
956
|
+
let componentExists = true;
|
|
957
|
+
try {
|
|
958
|
+
await lstat(application.dirPath);
|
|
959
|
+
} catch (error) {
|
|
960
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
961
|
+
componentExists = false;
|
|
962
|
+
}
|
|
963
|
+
if (componentExists) {
|
|
964
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
965
|
+
asidePath = join(asideStagingDir, `${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}`);
|
|
966
|
+
await rename(application.dirPath, asidePath);
|
|
967
|
+
transactionPaths.add(asidePath);
|
|
968
|
+
recoveryRecordPath = asidePath;
|
|
969
|
+
} else {
|
|
970
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
971
|
+
recoveryRecordPath = join(
|
|
972
|
+
asideStagingDir,
|
|
973
|
+
`${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}${PRIOR_ABSENT_RECORD_SUFFIX}`
|
|
974
|
+
);
|
|
975
|
+
await writeFile(recoveryRecordPath, '', { flag: 'wx', mode: 0o600 });
|
|
976
|
+
transactionPaths.add(recoveryRecordPath);
|
|
977
|
+
}
|
|
978
|
+
if (asidePath) application.isNewComponent = false;
|
|
979
|
+
|
|
980
|
+
try {
|
|
981
|
+
// The scratch dir for the pack-wrapper flatten has to be on the component root's filesystem, and
|
|
982
|
+
// is a transaction path so a crash mid-flatten is cleaned up with the rest.
|
|
983
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
984
|
+
const normalizeTempPath = await extractTarballInto(tarball, application.dirPath, asideStagingDir);
|
|
985
|
+
if (normalizeTempPath) transactionPaths.add(normalizeTempPath);
|
|
986
|
+
} catch (error) {
|
|
987
|
+
try {
|
|
988
|
+
await rollbackExtractedDirectory(application, asideStagingDir, asidePath, transactionPaths, false);
|
|
989
|
+
} catch (rollbackError) {
|
|
990
|
+
throw new AggregateError(
|
|
991
|
+
[error, rollbackError],
|
|
992
|
+
`Failed to extract ${application.name}: ${errorMessage(error)}; ` +
|
|
993
|
+
`also failed to restore its previous component directory: ${errorMessage(rollbackError)}`
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
throw error;
|
|
997
|
+
}
|
|
998
|
+
} finally {
|
|
999
|
+
if (!tarball.destroyed) tarball.destroy();
|
|
1000
|
+
if (shouldDeleteTarball && tarballPath) {
|
|
1001
|
+
await rm(tarballPath, { force: true }).catch((error) =>
|
|
1002
|
+
application.logger.warn(`Failed to remove temporary package ${tarballPath}:`, error)
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
let settled = false;
|
|
1008
|
+
const transaction: ExtractionTransaction = {
|
|
1009
|
+
async commit() {
|
|
1010
|
+
if (settled) return;
|
|
1011
|
+
const retiredMarkerPath = await retireExtractionAside(recoveryRecordPath);
|
|
1012
|
+
transactionPaths.add(retiredMarkerPath);
|
|
1013
|
+
settled = true;
|
|
1014
|
+
await cleanupExtractionPaths(application, asideStagingDir, transactionPaths);
|
|
1015
|
+
},
|
|
1016
|
+
async rollback() {
|
|
1017
|
+
if (settled) return;
|
|
1018
|
+
await rollbackExtractedDirectory(application, asideStagingDir, asidePath, transactionPaths, true);
|
|
1019
|
+
settled = true;
|
|
1020
|
+
},
|
|
1021
|
+
};
|
|
1022
|
+
if (deferCommit) return transaction;
|
|
1023
|
+
await transaction.commit();
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// Written into a candidate's deployment directory once its build and its validation have BOTH succeeded.
|
|
1027
|
+
// Roll-forward authority: recovery may activate an interrupted candidate only if this is present.
|
|
1028
|
+
// Every control file is dot-prefixed, and `isJoinableComponentName` rejects a leading dot: a deployment
|
|
1029
|
+
// directory holds the candidate tree under the COMPONENT'S name beside these, so an undotted name would
|
|
1030
|
+
// share that namespace. A component named `activation.json` would put its tree on the journal path, the
|
|
1031
|
+
// journal write would take EEXIST as "a retry of this activation", and the swap would proceed with no
|
|
1032
|
+
// journal to hold the legacy pass back; one named `unsettled` would make every settle throw on a
|
|
1033
|
+
// non-recursive `rm` of a directory. `assertApplicationConfig` rejects any name `isJoinableComponentName`
|
|
1034
|
+
// rejects, so the collision is unreachable from a root-config key as well as from a deploy.
|
|
1035
|
+
const CANDIDATE_COMPLETE_MARKER = '.complete';
|
|
1036
|
+
// Records activation intent beside the candidate, so recovery finishes or undoes the whole transaction —
|
|
1037
|
+
// tree and configuration together — instead of inferring intent from filesystem shape alone.
|
|
1038
|
+
const ACTIVATION_JOURNAL = '.activation.json';
|
|
1039
|
+
// The component this deployment directory belongs to, as plain text in its own file. Redundant with the
|
|
1040
|
+
// journal on purpose: after the swap the candidate has moved to the live path, so a journal that cannot be
|
|
1041
|
+
// parsed leaves nothing to infer the component from — and a failure keyed by deployment id fails NOTHING
|
|
1042
|
+
// closed, letting the component load over state nobody reconciled.
|
|
1043
|
+
const CANDIDATE_COMPONENT_FILE = '.component';
|
|
1044
|
+
// Written by main-thread recovery when it could not settle an activation whose journal is otherwise
|
|
1045
|
+
// well-formed. Workers cannot infer that case: a well-formed journal is indistinguishable from one belonging
|
|
1046
|
+
// to a deploy in flight, so without a record they would treat an unsettled component as healthy and load it.
|
|
1047
|
+
const UNSETTLED_MARKER = '.unsettled';
|
|
1048
|
+
// Everything the build decided that a later activation cannot re-derive: the root-config entry to publish,
|
|
1049
|
+
// whether the installation is opaque to metadata comparison, and the isolation intent that was admitted.
|
|
1050
|
+
// Written before `.complete`, so the marker vouches for it. An OPTIONAL record would not do: it could not
|
|
1051
|
+
// distinguish a payload build, which owns no root config, from a package build whose record was lost.
|
|
1052
|
+
const CANDIDATE_ARTIFACT_FILE = '.artifact.json';
|
|
1053
|
+
const ACTIVATION_JOURNAL_VERSION = 1;
|
|
1054
|
+
const ARTIFACT_DESCRIPTOR_VERSION = 1;
|
|
1055
|
+
const DEFAULT_STAGING_RETENTION_MAX_COUNT = 5;
|
|
1056
|
+
|
|
1057
|
+
/** `deployment_stagingRetention_maxCount`; 0 keeps none. Only a number or numeric string counts, so `true`/`[]`/blank cannot become "keep nothing". */
|
|
1058
|
+
export function getStagingRetentionMaxCount(): number {
|
|
1059
|
+
const configured = getConfigValue(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT);
|
|
1060
|
+
if (typeof configured !== 'number' && typeof configured !== 'string') return DEFAULT_STAGING_RETENTION_MAX_COUNT;
|
|
1061
|
+
if (typeof configured === 'string' && configured.trim() === '') return DEFAULT_STAGING_RETENTION_MAX_COUNT;
|
|
1062
|
+
const parsed = Number(configured);
|
|
1063
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : DEFAULT_STAGING_RETENTION_MAX_COUNT;
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
type DormantBuild = {
|
|
1067
|
+
deploymentDirPath: string;
|
|
1068
|
+
deploymentId: string;
|
|
1069
|
+
completedAt: number;
|
|
1070
|
+
};
|
|
1071
|
+
|
|
1072
|
+
async function presentOrAbsent(path: string): Promise<import('node:fs').Stats | undefined> {
|
|
1073
|
+
return lstat(path).catch((error: NodeJS.ErrnoException) => {
|
|
1074
|
+
if (error?.code === 'ENOENT') return undefined;
|
|
1075
|
+
throw error;
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
/**
|
|
1080
|
+
* A dormant build: complete, tree present, no journal. Activation writes `.complete` moments before its
|
|
1081
|
+
* journal under the owner's preparation lock, so only a read under that lock is a verdict. A stale
|
|
1082
|
+
* `.unsettled` makes it residue instead, since only removing the directory clears that marker for workers.
|
|
1083
|
+
* Only ENOENT is absence; any other read error propagates so the caller preserves the entry.
|
|
1084
|
+
*/
|
|
1085
|
+
async function dormantBuildAt(deploymentDirPath: string, owner: string): Promise<DormantBuild | undefined> {
|
|
1086
|
+
const complete = await presentOrAbsent(join(deploymentDirPath, CANDIDATE_COMPLETE_MARKER));
|
|
1087
|
+
if (!complete) return undefined;
|
|
1088
|
+
if (await presentOrAbsent(join(deploymentDirPath, UNSETTLED_MARKER))) return undefined;
|
|
1089
|
+
const tree = await presentOrAbsent(join(deploymentDirPath, owner));
|
|
1090
|
+
if (!tree || !(tree.isDirectory() || tree.isSymbolicLink())) return undefined;
|
|
1091
|
+
return { deploymentDirPath, deploymentId: basename(deploymentDirPath), completedAt: complete.mtimeMs };
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
/**
|
|
1095
|
+
* Remove the oldest dormant builds beyond `maxCount`. The caller must hold the component's preparation lock;
|
|
1096
|
+
* every catalogued build is re-derived under it before the kept set is chosen, so a catalog read unlocked
|
|
1097
|
+
* cannot hold or miss a slot. Never throws: a failure must neither fail a component closed nor replace a
|
|
1098
|
+
* deploy's own error.
|
|
1099
|
+
*
|
|
1100
|
+
* `pinnedDeploymentId` is never evicted. A delayed activation runs this preamble under the same lock it is
|
|
1101
|
+
* about to activate under, so without the pin retention would delete the artifact the request named —
|
|
1102
|
+
* immediately, when the knob is `0`. The pin is applied after the kept set is chosen, so a pinned build in
|
|
1103
|
+
* the eviction tail leaves `maxCount + 1` on disk for the life of the request; the next preamble that does
|
|
1104
|
+
* not pin it brings the count back down.
|
|
1105
|
+
*/
|
|
1106
|
+
export async function pruneDormantBuilds(
|
|
1107
|
+
componentName: string,
|
|
1108
|
+
builds: DormantBuild[],
|
|
1109
|
+
maxCount: number,
|
|
1110
|
+
pinnedDeploymentId?: string
|
|
1111
|
+
): Promise<void> {
|
|
1112
|
+
const current: DormantBuild[] = [];
|
|
1113
|
+
for (const build of builds) {
|
|
1114
|
+
try {
|
|
1115
|
+
const fresh = await dormantBuildAt(build.deploymentDirPath, componentName);
|
|
1116
|
+
if (fresh && !(await presentOrAbsent(join(build.deploymentDirPath, ACTIVATION_JOURNAL)))) current.push(fresh);
|
|
1117
|
+
} catch (error) {
|
|
1118
|
+
logger.warn(
|
|
1119
|
+
`Leaving deploy staging ${build.deploymentDirPath} out of retention; it could not be read:`,
|
|
1120
|
+
errorForLog(error)
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
const evictions = current
|
|
1125
|
+
.sort(
|
|
1126
|
+
(left, right) =>
|
|
1127
|
+
right.completedAt - left.completedAt ||
|
|
1128
|
+
(left.deploymentId < right.deploymentId ? -1 : left.deploymentId > right.deploymentId ? 1 : 0)
|
|
1129
|
+
)
|
|
1130
|
+
.slice(Math.max(0, maxCount))
|
|
1131
|
+
.filter((build) => build.deploymentId !== pinnedDeploymentId);
|
|
1132
|
+
for (const build of evictions) {
|
|
1133
|
+
try {
|
|
1134
|
+
await rm(build.deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
1135
|
+
logger.debug?.(
|
|
1136
|
+
`Pruned dormant staged build ${build.deploymentId} of ${componentName} beyond deployment_stagingRetention_maxCount=${maxCount}`
|
|
1137
|
+
);
|
|
1138
|
+
} catch (error) {
|
|
1139
|
+
logger.warn(
|
|
1140
|
+
`Could not prune dormant staged build ${build.deploymentDirPath} of ${componentName}; it remains beyond ` +
|
|
1141
|
+
`deployment_stagingRetention_maxCount=${maxCount}:`,
|
|
1142
|
+
errorForLog(error)
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
/** Every dormant build a component owns; an unreadable directory is left out and logged. */
|
|
1149
|
+
async function dormantBuildsOf(componentsRootDirPath: string, componentName: string): Promise<DormantBuild[]> {
|
|
1150
|
+
const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR);
|
|
1151
|
+
let deployments;
|
|
1152
|
+
try {
|
|
1153
|
+
deployments = await readdir(stagingRoot, { withFileTypes: true });
|
|
1154
|
+
} catch (error) {
|
|
1155
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
|
|
1156
|
+
throw error;
|
|
1157
|
+
}
|
|
1158
|
+
const builds: DormantBuild[] = [];
|
|
1159
|
+
for (const deployment of deployments) {
|
|
1160
|
+
if (!deployment.isDirectory()) continue;
|
|
1161
|
+
const deploymentDirPath = join(stagingRoot, deployment.name);
|
|
1162
|
+
try {
|
|
1163
|
+
if ((await candidateComponentName(deploymentDirPath)) !== componentName) continue;
|
|
1164
|
+
if (await presentOrAbsent(join(deploymentDirPath, ACTIVATION_JOURNAL))) continue;
|
|
1165
|
+
const build = await dormantBuildAt(deploymentDirPath, componentName);
|
|
1166
|
+
if (build) builds.push(build);
|
|
1167
|
+
} catch (error) {
|
|
1168
|
+
logger.warn(
|
|
1169
|
+
`Leaving deploy staging ${deploymentDirPath} out of retention; it could not be read:`,
|
|
1170
|
+
errorForLog(error)
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
return builds;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* Best-effort fsync of a directory. Best-effort by necessity — Node cannot fsync a directory on Windows —
|
|
1179
|
+
* which is why roll-forward requires journal + candidate + complete marker to all be observable: a lost
|
|
1180
|
+
* directory update then degrades to a roll back, never to a wrong decision. See DESIGN.md.
|
|
1181
|
+
*/
|
|
1182
|
+
async function syncDirectory(dirPath: string): Promise<void> {
|
|
1183
|
+
let handle;
|
|
1184
|
+
try {
|
|
1185
|
+
handle = await open(dirPath, 'r');
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
// Same split as `sync` below and as the file path: Windows cannot open a directory for fsync at all,
|
|
1188
|
+
// and a directory removed by cleanup is not a fault either — but an EIO opening it is.
|
|
1189
|
+
if (!isUnsupportedSync(error) && (error as NodeJS.ErrnoException)?.code !== 'ENOENT') throw error;
|
|
1190
|
+
logger.trace?.(`Directory sync of ${dirPath} unavailable: ${errorMessage(error)}`);
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
try {
|
|
1194
|
+
await handle.sync();
|
|
1195
|
+
} catch (error) {
|
|
1196
|
+
// A platform that will not sync directories is tolerated, a storage failure is not: suppressing
|
|
1197
|
+
// EIO/ENOSPC here would let a lost directory entry look durable. The `finally` closes the handle.
|
|
1198
|
+
if (!isUnsupportedSync(error)) throw error;
|
|
1199
|
+
logger.trace?.(`Directory sync of ${dirPath} unsupported: ${errorMessage(error)}`);
|
|
1200
|
+
} finally {
|
|
1201
|
+
// Swallowed: this runs outside any compensation block, so a rejecting close would surface as an
|
|
1202
|
+
// activation failure for something already best-effort.
|
|
1203
|
+
await handle.close().catch((error) => logger.trace?.(`Closing ${dirPath} failed: ${errorMessage(error)}`));
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/**
|
|
1208
|
+
* A rename changes an entry in BOTH parents, so both are synced: a surviving source entry reads as
|
|
1209
|
+
* "candidate still there" and would roll an already-completed activation forward twice.
|
|
1210
|
+
*/
|
|
1211
|
+
async function syncRenameParents(fromPath: string, toPath: string): Promise<void> {
|
|
1212
|
+
const parents = new Set([dirname(fromPath), dirname(toPath)]);
|
|
1213
|
+
for (const parent of parents) await syncDirectory(parent);
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// Deliberately NOT `EEXIST`/`ENOTEMPTY`/`ENOTDIR`/`EISDIR`: those say the destination exists, which
|
|
1217
|
+
// nothing here clears between attempts, so waiting on them would only delay reporting a tree something
|
|
1218
|
+
// recreated — the case `settleInterruptedActivation` fails closed rather than guessing.
|
|
1219
|
+
// `rollbackExtractedDirectory` does retry them, because its placeholder logic repairs the destination.
|
|
1220
|
+
const TRANSIENT_RENAME_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']);
|
|
1221
|
+
const RENAME_RETRY_BUDGET_MS = 5000;
|
|
1222
|
+
const RENAME_RETRY_INITIAL_DELAY_MS = 10;
|
|
1223
|
+
const RENAME_RETRY_MAX_DELAY_MS = 500;
|
|
1224
|
+
|
|
1225
|
+
/**
|
|
1226
|
+
* Rename, waiting out a holder that has not let go yet — on Windows a rename is refused outright while
|
|
1227
|
+
* anything still has a handle in the source tree.
|
|
1228
|
+
*
|
|
1229
|
+
* `onBackoff` replaces the sleep between attempts; `deadline` lets a rename it performs share this
|
|
1230
|
+
* call's budget instead of opening its own.
|
|
1231
|
+
*/
|
|
1232
|
+
async function renameThroughTransientHolder(
|
|
1233
|
+
fromPath: string,
|
|
1234
|
+
toPath: string,
|
|
1235
|
+
options: { onBackoff?: (delayMs: number, deadline: number) => Promise<void>; deadline?: number } = {}
|
|
1236
|
+
): Promise<void> {
|
|
1237
|
+
const deadline = options.deadline ?? performance.now() + RENAME_RETRY_BUDGET_MS;
|
|
1238
|
+
let delayMs = RENAME_RETRY_INITIAL_DELAY_MS;
|
|
1239
|
+
for (let attempts = 1; ; attempts++) {
|
|
1240
|
+
try {
|
|
1241
|
+
await rename(fromPath, toPath);
|
|
1242
|
+
if (attempts > 1) {
|
|
1243
|
+
logger.warn(`Renamed ${fromPath} to ${toPath} only on attempt ${attempts}; something was holding it`);
|
|
1244
|
+
}
|
|
1245
|
+
return;
|
|
1246
|
+
} catch (error) {
|
|
1247
|
+
const code = (error as NodeJS.ErrnoException).code ?? '';
|
|
1248
|
+
if (!TRANSIENT_RENAME_CODES.has(code)) throw error;
|
|
1249
|
+
if (performance.now() >= deadline) {
|
|
1250
|
+
// Which side was still there separates a holder on the source from a destination something
|
|
1251
|
+
// recreated, and neither survives on the rethrown error. A failed probe reports its own code:
|
|
1252
|
+
// an `EPERM` reading the destination is itself evidence, and calling it absent would send the
|
|
1253
|
+
// next investigation the wrong way.
|
|
1254
|
+
const state = async (path: string) =>
|
|
1255
|
+
lstat(path).then(
|
|
1256
|
+
() => 'present',
|
|
1257
|
+
(probeError) => (probeError as NodeJS.ErrnoException)?.code ?? 'unreadable'
|
|
1258
|
+
);
|
|
1259
|
+
logger.warn(
|
|
1260
|
+
`Could not rename ${fromPath} to ${toPath}: ${code} after ${attempts} attempts ` +
|
|
1261
|
+
`(source ${await state(fromPath)}, destination ${await state(toPath)})`
|
|
692
1262
|
);
|
|
1263
|
+
throw error;
|
|
693
1264
|
}
|
|
1265
|
+
if (options.onBackoff) await options.onBackoff(delayMs, deadline);
|
|
1266
|
+
else await delay(delayMs);
|
|
1267
|
+
delayMs = Math.min(delayMs * 2, RENAME_RETRY_MAX_DELAY_MS);
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/**
|
|
1273
|
+
* Write a control file so its final name NEVER exists with partial contents. Opening the final path with
|
|
1274
|
+
* `wx` publishes the directory entry before anything is written, so a crash in between leaves a zero-byte
|
|
1275
|
+
* file — which for the journal means "unreadable", failing a component closed over a deploy that had not
|
|
1276
|
+
* actually started. Contents land in a temp name, are fsynced, and only then renamed into place.
|
|
1277
|
+
*/
|
|
1278
|
+
async function writeControlFileDurably(filePath: string, contents: string): Promise<void> {
|
|
1279
|
+
const tempPath = `${filePath}.partial-${process.pid}-${randomUUID()}`;
|
|
1280
|
+
const handle = await open(tempPath, 'wx', 0o600);
|
|
1281
|
+
try {
|
|
1282
|
+
await handle.writeFile(contents, 'utf8');
|
|
1283
|
+
await handle.sync();
|
|
1284
|
+
} finally {
|
|
1285
|
+
await handle.close();
|
|
1286
|
+
}
|
|
1287
|
+
try {
|
|
1288
|
+
// `wx` on the temp name plus this rename keeps the EEXIST semantics callers rely on to detect a
|
|
1289
|
+
// retry of the same activation.
|
|
1290
|
+
await link(tempPath, filePath);
|
|
1291
|
+
} finally {
|
|
1292
|
+
await rm(tempPath, { force: true });
|
|
1293
|
+
}
|
|
1294
|
+
await syncDirectory(dirname(filePath));
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
type ActivationJournal = {
|
|
1298
|
+
v: number;
|
|
1299
|
+
component: string;
|
|
1300
|
+
candidateId: string;
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1303
|
+
function candidateCompleteMarkerPath(componentDirPath: string, deploymentId: string): string {
|
|
1304
|
+
return join(candidateDeploymentDirPath(componentDirPath, deploymentId), CANDIDATE_COMPLETE_MARKER);
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
function candidateComponentFilePath(componentDirPath: string, deploymentId: string): string {
|
|
1308
|
+
return join(candidateDeploymentDirPath(componentDirPath, deploymentId), CANDIDATE_COMPONENT_FILE);
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
function activationJournalPath(componentDirPath: string, deploymentId: string): string {
|
|
1312
|
+
return join(candidateDeploymentDirPath(componentDirPath, deploymentId), ACTIVATION_JOURNAL);
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
function candidateArtifactFilePath(componentDirPath: string, deploymentId: string): string {
|
|
1316
|
+
return join(candidateDeploymentDirPath(componentDirPath, deploymentId), CANDIDATE_ARTIFACT_FILE);
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
export type ArtifactDescriptor = {
|
|
1320
|
+
v: number;
|
|
1321
|
+
component: string;
|
|
1322
|
+
/** The root-config entry to publish with the activation; `null` for a payload build, which owns none. */
|
|
1323
|
+
rootConfig: Record<string, unknown> | null;
|
|
1324
|
+
installationIsOpaque: boolean;
|
|
1325
|
+
isolated: boolean;
|
|
1326
|
+
};
|
|
1327
|
+
|
|
1328
|
+
/**
|
|
1329
|
+
* Read and fully validate a staged artifact's descriptor. Every field is checked, not just the ones the
|
|
1330
|
+
* caller happens to use: a descriptor is activation input for a build this process did not make, possibly
|
|
1331
|
+
* not even on this node, so a partially-checked one is a way to activate under someone else's intent.
|
|
1332
|
+
* Absence is `undefined`; anything present but unusable throws, because a staged artifact that cannot
|
|
1333
|
+
* describe itself must be refused rather than activated under defaults.
|
|
1334
|
+
*/
|
|
1335
|
+
async function readArtifactDescriptor(
|
|
1336
|
+
deploymentDirPath: string,
|
|
1337
|
+
componentName: string
|
|
1338
|
+
): Promise<ArtifactDescriptor | undefined> {
|
|
1339
|
+
const descriptorPath = join(deploymentDirPath, CANDIDATE_ARTIFACT_FILE);
|
|
1340
|
+
// The artifact the caller named exists and is theirs, but does not describe a build this can activate —
|
|
1341
|
+
// a conflict with what is on disk, not a server fault.
|
|
1342
|
+
const unusable = (message: string) => new ClientError(message, 409);
|
|
1343
|
+
const raw = await readFile(descriptorPath, 'utf8').catch((error: NodeJS.ErrnoException) => {
|
|
1344
|
+
if (error?.code === 'ENOENT') return undefined;
|
|
1345
|
+
throw error;
|
|
1346
|
+
});
|
|
1347
|
+
if (raw === undefined) return undefined;
|
|
1348
|
+
let parsed: any;
|
|
1349
|
+
try {
|
|
1350
|
+
parsed = JSON.parse(raw);
|
|
1351
|
+
} catch (error) {
|
|
1352
|
+
throw unusable(`Artifact descriptor ${descriptorPath} is not readable JSON: ${errorMessage(error)}`);
|
|
1353
|
+
}
|
|
1354
|
+
if (!parsed || typeof parsed !== 'object' || parsed.v !== ARTIFACT_DESCRIPTOR_VERSION) {
|
|
1355
|
+
throw unusable(`Artifact descriptor ${descriptorPath} is version ${parsed?.v}, which this build cannot activate`);
|
|
1356
|
+
}
|
|
1357
|
+
if (!isJoinableComponentName(parsed.component) || parsed.component !== componentName) {
|
|
1358
|
+
throw unusable(
|
|
1359
|
+
`Artifact descriptor ${descriptorPath} names component '${parsed.component}', not '${componentName}'`
|
|
1360
|
+
);
|
|
1361
|
+
}
|
|
1362
|
+
if (typeof parsed.installationIsOpaque !== 'boolean' || typeof parsed.isolated !== 'boolean') {
|
|
1363
|
+
throw unusable(`Artifact descriptor ${descriptorPath} does not record its build's runtime decisions`);
|
|
1364
|
+
}
|
|
1365
|
+
if (parsed.rootConfig !== null && (typeof parsed.rootConfig !== 'object' || Array.isArray(parsed.rootConfig))) {
|
|
1366
|
+
throw unusable(`Artifact descriptor ${descriptorPath} does not record a root-config entry or its absence`);
|
|
1367
|
+
}
|
|
1368
|
+
// Admission reads one field and publication writes the other, so two authorities that disagree would
|
|
1369
|
+
// admit one isolation and then publish the opposite.
|
|
1370
|
+
if (parsed.rootConfig && Boolean(parsed.rootConfig.isolated) !== parsed.isolated) {
|
|
1371
|
+
throw unusable(
|
|
1372
|
+
`Artifact descriptor ${descriptorPath} admits isolated=${parsed.isolated} but publishes ` +
|
|
1373
|
+
`isolated=${Boolean(parsed.rootConfig.isolated)}`
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
return parsed as ArtifactDescriptor;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
const SEPARATORS_IN_LINK_TARGETS = process.platform === 'win32' ? /[\\/]/ : /\//;
|
|
1380
|
+
|
|
1381
|
+
// `node_modules/harper` and `node_modules/harperdb` are links the LOADER owns: it points them at the
|
|
1382
|
+
// running install on every non-root component load and repairs them when they are missing or stale. They
|
|
1383
|
+
// are outside the artifact by construction and by design, so they are the one external link a staged
|
|
1384
|
+
// artifact may carry.
|
|
1385
|
+
const LOADER_OWNED_LINKS = new Set(['harper', 'harperdb']);
|
|
1386
|
+
|
|
1387
|
+
/**
|
|
1388
|
+
* Reject a staged artifact that reaches outside itself.
|
|
1389
|
+
*
|
|
1390
|
+
* Certification fsyncs the tree but follows no links, and the post-swap relocation repair deliberately
|
|
1391
|
+
* leaves external targets alone — so a symlink into a directory this artifact does not own is a hole in
|
|
1392
|
+
* "activate exactly the bytes that were certified": the target can be edited, or replaced wholesale,
|
|
1393
|
+
* between the stage and the activation. An immediate deploy is not exposed to this, because certification
|
|
1394
|
+
* and activation happen within one call; the delay is what makes it reachable.
|
|
1395
|
+
*
|
|
1396
|
+
*/
|
|
1397
|
+
async function assertOwnedArtifactTree(
|
|
1398
|
+
candidateDirPath: string,
|
|
1399
|
+
componentName: string,
|
|
1400
|
+
action: 'stage' | 'activate' = 'stage'
|
|
1401
|
+
): Promise<void> {
|
|
1402
|
+
// The operator supplied a component that cannot be staged (400); or the artifact they named exists and is
|
|
1403
|
+
// theirs but is no longer what was certified (409). Neither is a server fault, and both reached the
|
|
1404
|
+
// operations handler as a bare 500 until a live run showed what that looks like to a caller.
|
|
1405
|
+
const refuse = (message: string) => new ClientError(message, action === 'stage' ? 400 : 409);
|
|
1406
|
+
const ownedRoot = await realpath(candidateDirPath);
|
|
1407
|
+
// The loader repairs the component's OWN `node_modules/harper`, not a copy nested inside a dependency,
|
|
1408
|
+
// so only that one path is exempt. Matching the name at any depth would let `dep/node_modules/harper`
|
|
1409
|
+
// point anywhere and still pass.
|
|
1410
|
+
const loaderOwnedDir = join(candidateDirPath, 'node_modules');
|
|
1411
|
+
const walk = async (dirPath: string): Promise<void> => {
|
|
1412
|
+
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
1413
|
+
for (const entry of entries) {
|
|
1414
|
+
const entryPath = join(dirPath, entry.name);
|
|
1415
|
+
if (entry.isDirectory()) {
|
|
1416
|
+
await walk(entryPath);
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
// Junctions report as symbolic links here, which is what makes this cover Windows.
|
|
1420
|
+
if (!entry.isSymbolicLink()) continue;
|
|
1421
|
+
if (LOADER_OWNED_LINKS.has(entry.name) && dirPath === loaderOwnedDir) continue;
|
|
1422
|
+
// An unresolvable link is rejected for the same reason a foreign one is: nothing certified what
|
|
1423
|
+
// it will resolve to by the time somebody activates it.
|
|
1424
|
+
const target = await realpath(entryPath).catch(() => undefined);
|
|
1425
|
+
if (target === undefined || (target !== ownedRoot && !target.startsWith(ownedRoot + sep))) {
|
|
1426
|
+
throw refuse(
|
|
1427
|
+
`Cannot ${action} ${componentName}: ${entryPath} links outside the build to ${target ?? 'a missing target'}, ` +
|
|
1428
|
+
`so the bytes activated later would not be the bytes this build certified`
|
|
1429
|
+
);
|
|
1430
|
+
}
|
|
1431
|
+
// `repairRelocatedDependencyLinks` re-points links after the swap, but it runs PAST THE COMMIT
|
|
1432
|
+
// POINT and can only warn — it logs and continues on a failed re-point, and skips a whole subtree
|
|
1433
|
+
// on EACCES/EMFILE — so a component could go live holding a link to a path that no longer exists
|
|
1434
|
+
// while the operation reports success. Staging fails closed instead. The cost: npm writes absolute
|
|
1435
|
+
// junctions under `node_modules` on Windows for a `file:`/workspace dependency, so such a component
|
|
1436
|
+
// deploys immediately but cannot be staged until its links are relative.
|
|
1437
|
+
const linkTarget = await readlink(entryPath);
|
|
1438
|
+
if (isAbsolute(linkTarget)) {
|
|
1439
|
+
throw refuse(
|
|
1440
|
+
`Cannot ${action} ${componentName}: ${entryPath} names its target inside the build by absolute path ` +
|
|
1441
|
+
`(${linkTarget}), which activation moves. Re-link it relatively — on Windows, npm ` +
|
|
1442
|
+
`writes absolute junctions for 'file:' and workspace dependencies, so those have to be relative ` +
|
|
1443
|
+
`before the component can be staged.`
|
|
1444
|
+
);
|
|
1445
|
+
}
|
|
1446
|
+
// Where the link ENDS UP is not enough: a target that leaves the candidate and comes back resolves
|
|
1447
|
+
// inside it today and somewhere else once activation renames the tree, because the same relative
|
|
1448
|
+
// expression is then evaluated from `components/<component>/…`. Counting `..` segments does not
|
|
1449
|
+
// catch it either, since an intermediate symlink (`up -> ..`) reduces depth without spelling it.
|
|
1450
|
+
// So every PREFIX of the walk is resolved, with symlinks followed as the filesystem will follow
|
|
1451
|
+
// them, and each one has to still be inside the candidate.
|
|
1452
|
+
let prefix = dirname(entryPath);
|
|
1453
|
+
// Only Windows treats a backslash as a separator. On POSIX it is an ordinary filename character, so
|
|
1454
|
+
// splitting on it there turns a link to the single legal entry `..\asset` — which resolves inside
|
|
1455
|
+
// the candidate and keeps resolving there after relocation — into `..` plus `asset`, and refuses a
|
|
1456
|
+
// component that never left its own tree.
|
|
1457
|
+
for (const segment of linkTarget.split(SEPARATORS_IN_LINK_TARGETS)) {
|
|
1458
|
+
if (segment === '' || segment === '.') continue;
|
|
1459
|
+
prefix = await realpath(join(prefix, segment)).catch(() => join(prefix, segment));
|
|
1460
|
+
if (prefix !== ownedRoot && !prefix.startsWith(ownedRoot + sep)) {
|
|
1461
|
+
throw refuse(
|
|
1462
|
+
`Cannot ${action} ${componentName}: ${entryPath} reaches ${prefix} on its way to ${linkTarget}, ` +
|
|
1463
|
+
`leaving the build — after activation moves the tree that path resolves somewhere else`
|
|
1464
|
+
);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
};
|
|
1469
|
+
await walk(candidateDirPath);
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
/** Record the build's decisions beside the candidate. Called before `.complete`, which vouches for it. */
|
|
1473
|
+
async function writeArtifactDescriptor(
|
|
1474
|
+
componentDirPath: string,
|
|
1475
|
+
deploymentId: string,
|
|
1476
|
+
descriptor: ArtifactDescriptor
|
|
1477
|
+
): Promise<void> {
|
|
1478
|
+
try {
|
|
1479
|
+
await writeControlFileDurably(
|
|
1480
|
+
candidateArtifactFilePath(componentDirPath, deploymentId),
|
|
1481
|
+
JSON.stringify(descriptor)
|
|
1482
|
+
);
|
|
1483
|
+
} catch (error) {
|
|
1484
|
+
// An existing descriptor belongs to this same artifact — the id is claimed exclusively, so nothing
|
|
1485
|
+
// else can have written one — which makes this a retry of its own stage rather than a conflict.
|
|
1486
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
/**
|
|
1491
|
+
* A component name safe to join onto the components root: no separator, no traversal, not dot-prefixed.
|
|
1492
|
+
* Applied to EVERY source of the name — the journal and the sidecar — because validating one and trusting
|
|
1493
|
+
* the other is how a corrupt record reaches an unrelated directory.
|
|
1494
|
+
*/
|
|
1495
|
+
function isJoinableComponentName(name: unknown): name is string {
|
|
1496
|
+
return (
|
|
1497
|
+
typeof name === 'string' &&
|
|
1498
|
+
name.length > 0 &&
|
|
1499
|
+
name === basename(name) &&
|
|
1500
|
+
name !== '.' &&
|
|
1501
|
+
name !== '..' &&
|
|
1502
|
+
!name.startsWith('.')
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
/**
|
|
1507
|
+
* Read an activation journal. Absent is `undefined` — no activation was attempted. Anything else THROWS:
|
|
1508
|
+
* a truncated or unknown-version journal is an interrupted activation whose intent cannot be read, and
|
|
1509
|
+
* both guesses are destructive (publish a rejected release, or discard a good one), so the component is
|
|
1510
|
+
* failed closed instead.
|
|
1511
|
+
*/
|
|
1512
|
+
async function readActivationJournal(journalPath: string): Promise<ActivationJournal | undefined> {
|
|
1513
|
+
let raw: string;
|
|
1514
|
+
try {
|
|
1515
|
+
raw = await readFile(journalPath, 'utf8');
|
|
1516
|
+
} catch (error) {
|
|
1517
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
|
1518
|
+
throw error;
|
|
1519
|
+
}
|
|
1520
|
+
let parsed: any;
|
|
1521
|
+
try {
|
|
1522
|
+
parsed = JSON.parse(raw);
|
|
1523
|
+
} catch (error) {
|
|
1524
|
+
throw new Error(`Activation journal ${journalPath} could not be parsed: ${errorMessage(error)}`);
|
|
1525
|
+
}
|
|
1526
|
+
if (parsed?.v !== ACTIVATION_JOURNAL_VERSION) {
|
|
1527
|
+
throw new Error(
|
|
1528
|
+
`Activation journal ${journalPath} has version ${JSON.stringify(parsed?.v)}, expected ${ACTIVATION_JOURNAL_VERSION}`
|
|
1529
|
+
);
|
|
1530
|
+
}
|
|
1531
|
+
if (!isJoinableComponentName(parsed.component) || typeof parsed.candidateId !== 'string') {
|
|
1532
|
+
throw new Error(`Activation journal ${journalPath} does not identify its component and candidate`);
|
|
1533
|
+
}
|
|
1534
|
+
// The journal must describe the directory it sits in. A syntactically valid journal naming someone
|
|
1535
|
+
// else's deployment would otherwise let recovery act on a component from the wrong record.
|
|
1536
|
+
if (parsed.candidateId !== basename(dirname(journalPath))) {
|
|
1537
|
+
throw new Error(
|
|
1538
|
+
`Activation journal ${journalPath} names candidate '${parsed.candidateId}', which is not its own deployment`
|
|
1539
|
+
);
|
|
1540
|
+
}
|
|
1541
|
+
return parsed as ActivationJournal;
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
/**
|
|
1545
|
+
* The deployment directory holding one candidate build: `<root>/.deploy-staging/<deploymentId>`.
|
|
1546
|
+
*
|
|
1547
|
+
* The id is asserted here rather than only at the request boundary because every caller funnels through
|
|
1548
|
+
* this one join, and the id is now operator-supplied (`deployment_id`) or replication-supplied
|
|
1549
|
+
* (`_deploymentId`). A traversal-bearing id would otherwise direct both the build and its removal outside
|
|
1550
|
+
* `.deploy-staging`.
|
|
1551
|
+
*/
|
|
1552
|
+
function candidateDeploymentDirPath(componentDirPath: string, deploymentId: string): string {
|
|
1553
|
+
if (
|
|
1554
|
+
typeof deploymentId !== 'string' ||
|
|
1555
|
+
deploymentId.length === 0 ||
|
|
1556
|
+
deploymentId !== basename(deploymentId) ||
|
|
1557
|
+
// `basename` returns these unchanged, so the comparison above admits both.
|
|
1558
|
+
deploymentId === '.' ||
|
|
1559
|
+
deploymentId === '..'
|
|
1560
|
+
) {
|
|
1561
|
+
throw new Error(`Deployment id '${deploymentId}' is not a single path segment`);
|
|
1562
|
+
}
|
|
1563
|
+
return join(dirname(componentDirPath), DEPLOY_STAGING_DIR, deploymentId);
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
/** Where a candidate build lives: `<root>/.deploy-staging/<deploymentId>/<component>`. */
|
|
1567
|
+
export function candidateApplicationPath(componentDirPath: string, deploymentId: string): string {
|
|
1568
|
+
return join(candidateDeploymentDirPath(componentDirPath, deploymentId), basename(componentDirPath));
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
function extractionStagingDirectory(componentDirPath: string): string {
|
|
1572
|
+
return join(dirname(componentDirPath), ASIDE_STAGING_DIR, basename(componentDirPath));
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
function retiredMarkerForAside(asidePath: string): string {
|
|
1576
|
+
return join(
|
|
1577
|
+
dirname(asidePath),
|
|
1578
|
+
`${RETIRED_ASIDE_PREFIX}${basename(asidePath).slice(IN_PROGRESS_ASIDE_PREFIX.length)}`
|
|
1579
|
+
);
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
async function retireExtractionAside(asidePath: string): Promise<string> {
|
|
1583
|
+
const retiredMarkerPath = retiredMarkerForAside(asidePath);
|
|
1584
|
+
try {
|
|
1585
|
+
await writeFile(retiredMarkerPath, '', { flag: 'wx', mode: 0o600 });
|
|
1586
|
+
} catch (error) {
|
|
1587
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
1588
|
+
}
|
|
1589
|
+
return retiredMarkerPath;
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function errorMessage(error: unknown): string {
|
|
1593
|
+
return error instanceof Error ? error.message : String(error);
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
export async function makeRollbackPlaceholderMovable(
|
|
1597
|
+
applicationDirPath: string,
|
|
1598
|
+
placeholderIdentity: { dev: bigint; ino: bigint } | undefined
|
|
1599
|
+
): Promise<void> {
|
|
1600
|
+
if (!placeholderIdentity) return;
|
|
1601
|
+
try {
|
|
1602
|
+
const current = await lstat(applicationDirPath, { bigint: true });
|
|
1603
|
+
if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) {
|
|
1604
|
+
await chmod(applicationDirPath, 0o700);
|
|
1605
|
+
}
|
|
1606
|
+
} catch (error) {
|
|
1607
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
async function identifyRollbackPlaceholder(
|
|
1612
|
+
applicationDirPath: string
|
|
1613
|
+
): Promise<{ dev: bigint; ino: bigint } | undefined> {
|
|
1614
|
+
const userId = process.getuid?.();
|
|
1615
|
+
if (process.platform === 'win32' || userId === undefined || userId === 0) return undefined;
|
|
1616
|
+
try {
|
|
1617
|
+
const current = await lstat(applicationDirPath, { bigint: true });
|
|
1618
|
+
const permissions = Number(current.mode) & 0o777;
|
|
1619
|
+
if (
|
|
1620
|
+
(current.isDirectory() || current.isFile()) &&
|
|
1621
|
+
current.uid === BigInt(userId) &&
|
|
1622
|
+
(permissions === 0 || (current.isDirectory() && (permissions === 0o100 || permissions === 0o300)))
|
|
1623
|
+
) {
|
|
1624
|
+
return { dev: current.dev, ino: current.ino };
|
|
1625
|
+
}
|
|
1626
|
+
} catch (error) {
|
|
1627
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
1628
|
+
}
|
|
1629
|
+
return undefined;
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/**
|
|
1633
|
+
* Create a hidden staging directory and confirm it is the one we created: a real directory rather than a
|
|
1634
|
+
* symlink or junction substituted underneath us, restricted to the owner. Re-checked at every use rather
|
|
1635
|
+
* than once per deploy, because the gap between checking and writing is the exploitable part.
|
|
1636
|
+
*/
|
|
1637
|
+
async function ensureSecureStagingDirectory(stagingDir: string): Promise<void> {
|
|
1638
|
+
await mkdir(stagingDir, { recursive: true, mode: 0o700 });
|
|
1639
|
+
const stagingStat = await lstat(stagingDir);
|
|
1640
|
+
if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) {
|
|
1641
|
+
throw new Error(`Component deploy staging path is not a directory: ${stagingDir}`);
|
|
1642
|
+
}
|
|
1643
|
+
if (process.platform !== 'win32' && (stagingStat.mode & 0o777) !== 0o700) {
|
|
1644
|
+
await chmod(stagingDir, 0o700).catch((error) =>
|
|
1645
|
+
logger.warn(`Could not restrict component deploy staging permissions for ${stagingDir}:`, errorForLog(error))
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
/**
|
|
1651
|
+
* Claim a deployment directory for this build, EXCLUSIVELY. The id is the public deployment id, which an
|
|
1652
|
+
* operator can repeat and a redelivered replication can repeat for them, so tolerating an existing
|
|
1653
|
+
* directory would let a replayed stage rewrite the bytes under an existing `.complete` and descriptor —
|
|
1654
|
+
* and a crash mid-rebuild would leave a partial tree that still reads as certified.
|
|
1655
|
+
*
|
|
1656
|
+
* The caller holds the component's preparation lock, which is what makes the EEXIST verdicts sound: no
|
|
1657
|
+
* other preparation of THIS component is running, and a directory belonging to another component is not
|
|
1658
|
+
* this lock's to touch.
|
|
1659
|
+
*/
|
|
1660
|
+
async function claimDeploymentDirectory(deploymentDirPath: string, componentName: string): Promise<void> {
|
|
1661
|
+
// Every refusal below is a conflict over an id that already exists, which is the caller's to resolve by
|
|
1662
|
+
// naming a different one — not a server fault, and not the 500 a bare Error reaches the caller as.
|
|
1663
|
+
const taken = (message: string) => new ClientError(message, 409);
|
|
1664
|
+
try {
|
|
1665
|
+
await mkdir(deploymentDirPath, { mode: 0o700 });
|
|
1666
|
+
} catch (error) {
|
|
1667
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
1668
|
+
const owner = await candidateComponentName(deploymentDirPath);
|
|
1669
|
+
if (owner !== undefined && owner !== componentName) {
|
|
1670
|
+
throw taken(
|
|
1671
|
+
`Deployment id ${basename(deploymentDirPath)} already holds a build of '${owner}'; a deployment id ` +
|
|
1672
|
+
`names one artifact for its lifetime`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
// Ownership has to be POSITIVE to reclaim, and EMPTINESS IS NOT A VERDICT. The sidecar below is
|
|
1676
|
+
// written as part of the claim, so a directory naming nobody is another component between its own
|
|
1677
|
+
// mkdir and that write, and a deploy can hold a nearly empty one for minutes while it resolves and
|
|
1678
|
+
// packs. Nothing on disk separates that from a claim that got no further, and the lock that
|
|
1679
|
+
// serializes it is not this one, so only the exclusive create may conclude the id is free: the
|
|
1680
|
+
// directory can also have been discarded by a failed build between the first mkdir and this read,
|
|
1681
|
+
// and another component can claim it in that same gap.
|
|
1682
|
+
if (owner === undefined) {
|
|
1683
|
+
await mkdir(deploymentDirPath, { mode: 0o700 }).catch((retry: NodeJS.ErrnoException) => {
|
|
1684
|
+
if (retry?.code !== 'EEXIST') throw retry;
|
|
1685
|
+
throw taken(
|
|
1686
|
+
`Deployment id ${basename(deploymentDirPath)} is already claimed by a build that has not named its ` +
|
|
1687
|
+
`component yet. If no deploy of any component is in flight, that directory is abandoned and has ` +
|
|
1688
|
+
`to be removed by hand; deploying again without a deployment_id mints a fresh id`
|
|
1689
|
+
);
|
|
1690
|
+
});
|
|
1691
|
+
} else {
|
|
1692
|
+
if (await presentOrAbsent(join(deploymentDirPath, CANDIDATE_COMPLETE_MARKER))) {
|
|
1693
|
+
throw taken(
|
|
1694
|
+
`Deployment id ${basename(deploymentDirPath)} already holds a completed build of '${componentName}'; ` +
|
|
1695
|
+
`deploy it with deployment_id, or deploy again to build a new one`
|
|
1696
|
+
);
|
|
1697
|
+
}
|
|
1698
|
+
// This component's own preparation lock serializes the claim and nothing certified it, so nothing
|
|
1699
|
+
// is lost by rebuilding over it.
|
|
1700
|
+
await rm(deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
1701
|
+
await mkdir(deploymentDirPath, { mode: 0o700 });
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
const claimed = await lstat(deploymentDirPath);
|
|
1705
|
+
if (!claimed.isDirectory() || claimed.isSymbolicLink()) {
|
|
1706
|
+
throw new Error(`Component deploy staging path is not a directory: ${deploymentDirPath}`);
|
|
1707
|
+
}
|
|
1708
|
+
if (process.platform !== 'win32' && (claimed.mode & 0o777) !== 0o700) {
|
|
1709
|
+
await chmod(deploymentDirPath, 0o700).catch((error) =>
|
|
1710
|
+
logger.warn(
|
|
1711
|
+
`Could not restrict component deploy staging permissions for ${deploymentDirPath}:`,
|
|
1712
|
+
errorForLog(error)
|
|
1713
|
+
)
|
|
1714
|
+
);
|
|
1715
|
+
}
|
|
1716
|
+
await publishClaimOwnership(deploymentDirPath, componentName);
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
/**
|
|
1720
|
+
* Name the component that owns a deployment directory THIS CALL created, and take the directory back if that
|
|
1721
|
+
* cannot be recorded.
|
|
1722
|
+
*
|
|
1723
|
+
* Ownership is published as part of the claim rather than at certification, because resolving and packing can
|
|
1724
|
+
* take minutes and until a tree exists to infer an owner from, a directory answering to nobody is one another
|
|
1725
|
+
* component will take for abandoned. Unattributed is a permanent refusal, though, so a failure here — a full
|
|
1726
|
+
* disk, an EIO on the temp write or its sync — would burn this deployment id for good, every retry refused by
|
|
1727
|
+
* its own wreckage. The removal is scoped to the directory this invocation made, and deliberately not
|
|
1728
|
+
* broadened to one found by EEXIST: that directory may be another claimant's, and removing it is the race the
|
|
1729
|
+
* refusal exists to prevent. Best-effort, because the claim failure is what the caller needs to see.
|
|
1730
|
+
*
|
|
1731
|
+
* `write` is a parameter so the failure is testable without a filesystem that can be made to fail on exactly
|
|
1732
|
+
* this write and nothing else.
|
|
1733
|
+
*/
|
|
1734
|
+
export async function publishClaimOwnership(
|
|
1735
|
+
deploymentDirPath: string,
|
|
1736
|
+
componentName: string,
|
|
1737
|
+
write: (filePath: string, contents: string) => Promise<void> = writeControlFileDurably
|
|
1738
|
+
): Promise<void> {
|
|
1739
|
+
try {
|
|
1740
|
+
await write(join(deploymentDirPath, CANDIDATE_COMPONENT_FILE), componentName);
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
await rm(deploymentDirPath, { recursive: true, force: true }).catch((cleanupError) =>
|
|
1743
|
+
logger.warn(
|
|
1744
|
+
`Could not remove the deployment directory ${deploymentDirPath} after failing to publish its ` +
|
|
1745
|
+
`ownership; the id stays unusable until it is removed:`,
|
|
1746
|
+
errorForLog(cleanupError)
|
|
1747
|
+
)
|
|
1748
|
+
);
|
|
1749
|
+
throw error;
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
async function ensureExtractionStagingDirectory(asideStagingDir: string): Promise<void> {
|
|
1754
|
+
for (const stagingDir of [dirname(asideStagingDir), asideStagingDir]) {
|
|
1755
|
+
await ensureSecureStagingDirectory(stagingDir);
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
/** The single component directory inside a candidate deployment directory, when there is exactly one. */
|
|
1760
|
+
async function candidateComponentName(deploymentDirPath: string): Promise<string | undefined> {
|
|
1761
|
+
// The sidecar first: it is the only source that still works once the candidate has been renamed to the
|
|
1762
|
+
// live path, which is exactly when an unreadable journal would otherwise be unattributable.
|
|
1763
|
+
// Only ENOENT is absence. Swallowing every error here reported "unowned", which is a licence to act:
|
|
1764
|
+
// the worker verdict dropped a failure and loaded a component main had failed closed, and a deploy
|
|
1765
|
+
// skipped a journaled activation it owns and stalled the component in the legacy pass instead.
|
|
1766
|
+
const recorded = await readFile(join(deploymentDirPath, CANDIDATE_COMPONENT_FILE), 'utf8').catch(
|
|
1767
|
+
(error: NodeJS.ErrnoException) => {
|
|
1768
|
+
if (error?.code === 'ENOENT') return '';
|
|
1769
|
+
throw error;
|
|
1770
|
+
}
|
|
1771
|
+
);
|
|
1772
|
+
const named = recorded.trim();
|
|
1773
|
+
if (isJoinableComponentName(named)) return named;
|
|
1774
|
+
const entries = await readdir(deploymentDirPath, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
|
|
1775
|
+
if (error?.code === 'ENOENT') return [];
|
|
1776
|
+
throw error;
|
|
1777
|
+
});
|
|
1778
|
+
// Symlinks count: a `file:<directory>` candidate is deliberately a link, and activation already accepts
|
|
1779
|
+
// one. Filtering to real directories here left those candidates with no owner, so residue removal took
|
|
1780
|
+
// no lock and could delete a build in flight.
|
|
1781
|
+
//
|
|
1782
|
+
// Validated, because this infers an owner from a NAME. A directory-shaped control file — a corrupt
|
|
1783
|
+
// `.activation.json` that is a directory — would otherwise be returned as the owning component, which
|
|
1784
|
+
// both licenses a restore against it and is a name no component can have.
|
|
1785
|
+
const components = entries.filter(
|
|
1786
|
+
(entry) => (entry.isDirectory() || entry.isSymbolicLink()) && isJoinableComponentName(entry.name)
|
|
1787
|
+
);
|
|
1788
|
+
return components.length === 1 ? components[0].name : undefined;
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
/**
|
|
1792
|
+
* The deployment directory of an activation journal this component still owns, if any.
|
|
1793
|
+
*
|
|
1794
|
+
* The journal is the authority for an interrupted activation. The legacy `.deploy-aside` pass would
|
|
1795
|
+
* otherwise restore the displaced tree over a candidate a completed activation already renamed live, so it
|
|
1796
|
+
* consults this before restoring rather than relying on being sequenced after settlement: a worker
|
|
1797
|
+
* auto-restarted mid-activation reaches it with no settlement in front of it, and settlement that FAILS
|
|
1798
|
+
* deliberately keeps the journal for the next start while the same boot carries on into the legacy pass.
|
|
1799
|
+
*/
|
|
1800
|
+
async function journaledDeploymentForComponent(
|
|
1801
|
+
componentsRootDirPath: string,
|
|
1802
|
+
componentName: string
|
|
1803
|
+
): Promise<string | undefined> {
|
|
1804
|
+
const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR);
|
|
1805
|
+
let deployments;
|
|
1806
|
+
try {
|
|
1807
|
+
deployments = await readdir(stagingRoot, { withFileTypes: true });
|
|
1808
|
+
} catch (error) {
|
|
1809
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
|
1810
|
+
throw error;
|
|
1811
|
+
}
|
|
1812
|
+
for (const deployment of deployments) {
|
|
1813
|
+
if (!deployment.isDirectory()) continue;
|
|
1814
|
+
const deploymentDirPath = join(stagingRoot, deployment.name);
|
|
1815
|
+
const journalPath = join(deploymentDirPath, ACTIVATION_JOURNAL);
|
|
1816
|
+
// NOTHING is swallowed here. This is the gate that authorizes restoring an old tree over what may be
|
|
1817
|
+
// a committed candidate, so "could not tell" has to fail closed — treating an unreadable deployment
|
|
1818
|
+
// as "no journal for this component" is exactly the clobber the journal exists to prevent. The blast
|
|
1819
|
+
// radius is narrow because the gate is only consulted where a restorable record already exists.
|
|
1820
|
+
//
|
|
1821
|
+
// Presence, not parseability: an unreadable journal is precisely the ambiguous case.
|
|
1822
|
+
const journaled = await lstat(journalPath).then(
|
|
1823
|
+
() => true,
|
|
1824
|
+
(error) => {
|
|
1825
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
|
1826
|
+
throw error;
|
|
1827
|
+
}
|
|
1828
|
+
);
|
|
1829
|
+
if (!journaled) continue;
|
|
1830
|
+
// EITHER name blocks, while settlement acts only when both agree. The destructive step takes the
|
|
1831
|
+
// conservative union; the corrective one takes the precise intersection, so a journal whose two
|
|
1832
|
+
// attributions disagree stalls the restore instead of licensing it, and startup recovery — which
|
|
1833
|
+
// keys on the journal — is what clears it. The sidecar also covers a component legitimately named
|
|
1834
|
+
// `component`, whose candidate path collides with the sidecar's and makes every sidecar read fail.
|
|
1835
|
+
const journalOwner = (await readActivationJournal(journalPath).catch(() => undefined))?.component;
|
|
1836
|
+
if (journalOwner === componentName) return deploymentDirPath;
|
|
1837
|
+
const sidecarOwner = await candidateComponentName(deploymentDirPath);
|
|
1838
|
+
if (sidecarOwner === componentName) return deploymentDirPath;
|
|
1839
|
+
// A journal nobody can attribute blocks EVERY component. It is a rare, genuinely broken state — an
|
|
1840
|
+
// unparseable journal whose deployment no longer holds a tree to infer from — and the alternative is
|
|
1841
|
+
// letting the restore proceed against a candidate this journal may well have committed. There is no
|
|
1842
|
+
// automated way out of it, by construction: nothing on disk says which component it belongs to. The
|
|
1843
|
+
// error the caller raises names the directory an operator has to resolve.
|
|
1844
|
+
if (journalOwner === undefined && sidecarOwner === undefined) return deploymentDirPath;
|
|
1845
|
+
}
|
|
1846
|
+
return undefined;
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
/**
|
|
1850
|
+
* The components a deployment's evidence implicates, when its journal and its ownership sidecar disagree
|
|
1851
|
+
* about whose it is — or `undefined` when there is no disagreement to act on.
|
|
1852
|
+
*
|
|
1853
|
+
* A sidecar that cannot answer is NOT disagreement: the journal names its own component and can settle it,
|
|
1854
|
+
* so an unreadable sidecar must not block that. Only a sidecar that reads and names something else is the
|
|
1855
|
+
* wedge, and then BOTH names are returned, sidecar first, because both are stuck — see
|
|
1856
|
+
* `splitAttributionError`.
|
|
1857
|
+
*
|
|
1858
|
+
* Separated from either call site so the decision is testable on its own: the paths that reach it include
|
|
1859
|
+
* one that requires a journal to appear between two reads under a lock, which no test can stage.
|
|
1860
|
+
*/
|
|
1861
|
+
export function splitAttributionOwners(
|
|
1862
|
+
journalOwner: string,
|
|
1863
|
+
sidecarOwner: string | undefined
|
|
1864
|
+
): [string, string] | undefined {
|
|
1865
|
+
if (sidecarOwner === undefined || sidecarOwner === journalOwner) return undefined;
|
|
1866
|
+
return [sidecarOwner, journalOwner];
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
/**
|
|
1870
|
+
* The error for a deployment its journal and its ownership sidecar attribute to different components.
|
|
1871
|
+
*
|
|
1872
|
+
* Both names are wedged, so both are failed: the restore gate takes the union and blocks the sidecar's
|
|
1873
|
+
* component, while settlement needs the intersection and so can never clear the journal owner's. Neither
|
|
1874
|
+
* component's own deploy can resolve it, which is why the message names the directory to remove.
|
|
1875
|
+
*/
|
|
1876
|
+
function splitAttributionError(deploymentDirPath: string, journalOwner: string, sidecarOwner: string): Error {
|
|
1877
|
+
return new Error(
|
|
1878
|
+
`Deploy staging ${deploymentDirPath} is attributed to two different components: its journal names ` +
|
|
1879
|
+
`'${journalOwner}' and its sidecar names '${sidecarOwner}'. Neither can settle it; remove that ` +
|
|
1880
|
+
`directory once you have determined which tree is current.`
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
/** In-progress rollback records in a component's aside directory, newest first. */
|
|
1885
|
+
async function inProgressAsideRecords(asideStagingDir: string): Promise<string[]> {
|
|
1886
|
+
// ENOENT is "no aside directory yet"; anything else would report "no records" and let roll-forward
|
|
1887
|
+
// remove the journal while the records it should have retired are still there and still authoritative.
|
|
1888
|
+
const entries = await readdir(asideStagingDir, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
|
|
1889
|
+
if (error?.code === 'ENOENT') return [];
|
|
1890
|
+
throw error;
|
|
1891
|
+
});
|
|
1892
|
+
// Retired records excluded, the same rule `recoverOrCleanupStaleExtractionPaths` applies. A record whose
|
|
1893
|
+
// retire succeeded but whose best-effort sweep did not is settled, not displaced — counting it would let
|
|
1894
|
+
// an ordinary pre-swap state look like the "live path recreated" ambiguity and fail a healthy component
|
|
1895
|
+
// closed with an operator-only exit.
|
|
1896
|
+
const entryNames = new Set(entries.map((entry) => entry.name));
|
|
1897
|
+
return entries
|
|
1898
|
+
.filter(
|
|
1899
|
+
(entry) =>
|
|
1900
|
+
entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX) &&
|
|
1901
|
+
!entryNames.has(`${RETIRED_ASIDE_PREFIX}${entry.name.slice(IN_PROGRESS_ASIDE_PREFIX.length)}`)
|
|
1902
|
+
)
|
|
1903
|
+
.map((entry) => join(asideStagingDir, entry.name))
|
|
1904
|
+
.sort()
|
|
1905
|
+
.reverse();
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
/**
|
|
1909
|
+
* Settle journaled activations for ONE component, and bound its dormant staged builds, assuming the caller
|
|
1910
|
+
* already holds its preparation lock.
|
|
1911
|
+
*
|
|
1912
|
+
* Exists because the journal-first rule has to hold at every entry point, not just startup. A deploy runs
|
|
1913
|
+
* `recoverOrCleanupStaleExtractionPaths` first. After an activation whose retirement failed, the aside
|
|
1914
|
+
* still names the DISPLACED tree, so restoring it would put the old version back over the new one. That
|
|
1915
|
+
* pass refuses to restore against a surviving journal, but refusing is a stalled component; settling first
|
|
1916
|
+
* is what lets the deploy proceed.
|
|
1917
|
+
*/
|
|
1918
|
+
async function settleStagingForComponent(
|
|
1919
|
+
componentsRootDirPath: string,
|
|
1920
|
+
componentName: string,
|
|
1921
|
+
pinnedDeploymentId?: string
|
|
1922
|
+
): Promise<void> {
|
|
1923
|
+
const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR);
|
|
1924
|
+
let deployments;
|
|
1925
|
+
try {
|
|
1926
|
+
deployments = await readdir(stagingRoot, { withFileTypes: true });
|
|
1927
|
+
} catch (error) {
|
|
1928
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
|
|
1929
|
+
throw error;
|
|
1930
|
+
}
|
|
1931
|
+
const dormant: DormantBuild[] = [];
|
|
1932
|
+
for (const deployment of deployments) {
|
|
1933
|
+
if (!deployment.isDirectory()) continue;
|
|
1934
|
+
const deploymentDirPath = join(stagingRoot, deployment.name);
|
|
1935
|
+
// Ownership BEFORE parsing. Reading every journal first meant a truncated journal belonging to another
|
|
1936
|
+
// component threw here — blocking the deploy of a healthy component because an unrelated one is
|
|
1937
|
+
// broken. The sidecar names the owner without parsing anything, and a deployment that does not name
|
|
1938
|
+
// this component is none of this deploy's business; startup recovery reports it instead.
|
|
1939
|
+
//
|
|
1940
|
+
// An ownership read that FAILS is the same situation, and skipping is safe here specifically because
|
|
1941
|
+
// settlement is the corrective half: if that entry does turn out to be this component's, the restore
|
|
1942
|
+
// gate — which takes the union and fails closed on anything it cannot attribute — is what stops the
|
|
1943
|
+
// legacy pass acting on it. Failing the deploy instead lets one unreadable sibling block every
|
|
1944
|
+
// neighbour's deploys, which is the outage this ordering exists to prevent.
|
|
1945
|
+
let owner: string | undefined;
|
|
1946
|
+
let ownerUnreadable = false;
|
|
1947
|
+
try {
|
|
1948
|
+
owner = await candidateComponentName(deploymentDirPath);
|
|
1949
|
+
} catch (error) {
|
|
1950
|
+
ownerUnreadable = true;
|
|
1951
|
+
logger.trace?.(
|
|
1952
|
+
`Ownership of ${deploymentDirPath} is unreadable while settling ${componentName}, falling back to ` +
|
|
1953
|
+
`its journal: ${errorMessage(error)}`
|
|
1954
|
+
);
|
|
1955
|
+
}
|
|
1956
|
+
if (!ownerUnreadable && owner !== componentName) continue;
|
|
1957
|
+
const journal = await readActivationJournal(join(deploymentDirPath, ACTIVATION_JOURNAL));
|
|
1958
|
+
// A journal-less directory is what a SUCCESSFUL settlement leaves when its best-effort sweep fails, so
|
|
1959
|
+
// this must not fail closed. An unattributable *activation* still does: `readActivationJournal` throws
|
|
1960
|
+
// on a journal that exists but cannot be read.
|
|
1961
|
+
if (!journal) {
|
|
1962
|
+
if (ownerUnreadable) continue;
|
|
1963
|
+
try {
|
|
1964
|
+
const build = await dormantBuildAt(deploymentDirPath, componentName);
|
|
1965
|
+
if (build) dormant.push(build);
|
|
1966
|
+
} catch (error) {
|
|
1967
|
+
logger.warn(
|
|
1968
|
+
`Leaving staged ${componentName} build ${deploymentDirPath} out of retention; it could not be read:`,
|
|
1969
|
+
errorForLog(error)
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
continue;
|
|
1973
|
+
}
|
|
1974
|
+
// The journal decides, not the sidecar. Skipping on an unreadable sidecar alone would leave this
|
|
1975
|
+
// component's own unsettled activation in place while a new deploy proceeded over it, and an
|
|
1976
|
+
// activation interrupted before B1 has no rollback record for the restore gate to catch.
|
|
1977
|
+
if (journal.component !== componentName) continue;
|
|
1978
|
+
await settleInterruptedActivation(componentsRootDirPath, deploymentDirPath, journal);
|
|
1979
|
+
}
|
|
1980
|
+
const maxCount = getStagingRetentionMaxCount();
|
|
1981
|
+
if (dormant.length > maxCount) await pruneDormantBuilds(componentName, dormant, maxCount, pinnedDeploymentId);
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
/**
|
|
1985
|
+
* Components that on-disk evidence says were left in a state nobody settled — determined READ-ONLY, so any
|
|
1986
|
+
* thread can reach the same verdict.
|
|
1987
|
+
*
|
|
1988
|
+
* Recovery runs on the main thread only, but the components it could not settle still have to be failed
|
|
1989
|
+
* closed on the workers that actually serve them, and a worker cannot be handed main's verdict: it boots
|
|
1990
|
+
* through its own `loadRootComponents(true)`, potentially before main finished. Recovery deliberately KEEPS
|
|
1991
|
+
* the evidence for anything it could not settle, so a worker can read it instead of being told.
|
|
1992
|
+
*
|
|
1993
|
+
* Only unambiguous evidence counts. A well-formed journal is NOT evidence — every healthy deploy has one
|
|
1994
|
+
* in flight — so this reports a journal that cannot be read at all (corrupt, unknown version, or naming
|
|
1995
|
+
* something other than its own deployment), which no in-flight deploy ever produces.
|
|
1996
|
+
*/
|
|
1997
|
+
export async function unsettleableComponentsFromDisk(componentsRootDirPath: string): Promise<Map<string, Error>> {
|
|
1998
|
+
const unsettleable = new Map<string, Error>();
|
|
1999
|
+
const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR);
|
|
2000
|
+
let deployments;
|
|
2001
|
+
try {
|
|
2002
|
+
deployments = await readdir(stagingRoot, { withFileTypes: true });
|
|
2003
|
+
} catch (error) {
|
|
2004
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return unsettleable;
|
|
2005
|
+
throw error;
|
|
2006
|
+
}
|
|
2007
|
+
for (const deployment of deployments) {
|
|
2008
|
+
if (!deployment.isDirectory()) continue;
|
|
2009
|
+
const deploymentDirPath = join(stagingRoot, deployment.name);
|
|
2010
|
+
// Per deployment. `candidateComponentName` propagates every non-ENOENT error now, and this pass runs
|
|
2011
|
+
// where the CALLER only warns — so one unreadable deployment escaping here would drop the verdict for
|
|
2012
|
+
// every other component and let each of them load with no evidence checked at all.
|
|
2013
|
+
try {
|
|
2014
|
+
await verdictFor(deploymentDirPath, deployment.name, unsettleable);
|
|
2015
|
+
} catch (error) {
|
|
2016
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
2017
|
+
if (!unsettleable.has(deployment.name)) unsettleable.set(deployment.name, failure);
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
return unsettleable;
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
/** One deployment's read-only verdict. Throws rather than guessing; the caller scopes that to this entry. */
|
|
2024
|
+
async function verdictFor(
|
|
2025
|
+
deploymentDirPath: string,
|
|
2026
|
+
deploymentName: string,
|
|
2027
|
+
unsettleable: Map<string, Error>
|
|
2028
|
+
): Promise<void> {
|
|
2029
|
+
// Recorded by main when it failed to settle a well-formed journal. Checked first, because that case is
|
|
2030
|
+
// invisible to a worker otherwise.
|
|
2031
|
+
//
|
|
2032
|
+
// Only ENOENT is absence. A marker that exists but cannot be read (EIO, EACCES) must not be taken as
|
|
2033
|
+
// "no verdict" — that classifies the well-formed journal beside it as a healthy in-flight deploy and
|
|
2034
|
+
// lets the worker load a component main failed closed.
|
|
2035
|
+
let recorded: string | undefined;
|
|
2036
|
+
try {
|
|
2037
|
+
recorded = await readFile(join(deploymentDirPath, UNSETTLED_MARKER), 'utf8');
|
|
2038
|
+
} catch (error) {
|
|
2039
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
|
2040
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
2041
|
+
return record(unsettleable, await attribute(deploymentDirPath, deploymentName), failure);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
if (recorded !== undefined) {
|
|
2045
|
+
const component = await attribute(deploymentDirPath, deploymentName);
|
|
2046
|
+
return record(
|
|
2047
|
+
unsettleable,
|
|
2048
|
+
component,
|
|
2049
|
+
new Error(recorded.trim() || `Activation of ${component} could not be settled`)
|
|
2050
|
+
);
|
|
2051
|
+
}
|
|
2052
|
+
try {
|
|
2053
|
+
await readActivationJournal(join(deploymentDirPath, ACTIVATION_JOURNAL));
|
|
2054
|
+
} catch (error) {
|
|
2055
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
2056
|
+
record(unsettleable, await attribute(deploymentDirPath, deploymentName), failure);
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
/**
|
|
2061
|
+
* Who a deployment's evidence belongs to. Falls back to the deployment id when nothing names it: a verdict
|
|
2062
|
+
* attributed to nothing is a verdict nobody acts on, and the id is at least something an operator can find
|
|
2063
|
+
* on disk. A read that FAILS is not "nothing names it" — that propagates, and the caller records it against
|
|
2064
|
+
* the id, so an unreadable deployment is reported rather than dropped.
|
|
2065
|
+
*/
|
|
2066
|
+
async function attribute(deploymentDirPath: string, deploymentName: string): Promise<string> {
|
|
2067
|
+
return (await candidateComponentName(deploymentDirPath)) ?? deploymentName;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
function record(unsettleable: Map<string, Error>, component: string, failure: Error): void {
|
|
2071
|
+
if (!unsettleable.has(component)) unsettleable.set(component, failure);
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
/**
|
|
2075
|
+
* Settle activations a crash interrupted, before anything loads. Runs at startup on main and on every
|
|
2076
|
+
* worker — a worker can be respawned mid-activation, long after main's pass.
|
|
2077
|
+
*
|
|
2078
|
+
* Returns failures keyed by COMPONENT so the caller can fail exactly those closed and still load every
|
|
2079
|
+
* healthy sibling — a single unreadable journal must not take down the whole node, and must not let a
|
|
2080
|
+
* component load over state nobody reconciled.
|
|
2081
|
+
*
|
|
2082
|
+
*/
|
|
2083
|
+
export async function recoverInterruptedActivations(componentsRootDirPath: string): Promise<Map<string, Error>> {
|
|
2084
|
+
const failures = new Map<string, Error>();
|
|
2085
|
+
const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR);
|
|
2086
|
+
let deployments;
|
|
2087
|
+
try {
|
|
2088
|
+
deployments = await readdir(stagingRoot, { withFileTypes: true });
|
|
2089
|
+
} catch (error) {
|
|
2090
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return failures;
|
|
2091
|
+
throw error;
|
|
2092
|
+
}
|
|
2093
|
+
const dormant = new Map<string, DormantBuild[]>();
|
|
2094
|
+
const catalogue = (owner: string, build: DormantBuild) => {
|
|
2095
|
+
const builds = dormant.get(owner);
|
|
2096
|
+
if (builds) builds.push(build);
|
|
2097
|
+
else dormant.set(owner, [build]);
|
|
2098
|
+
};
|
|
2099
|
+
|
|
2100
|
+
for (const deployment of deployments) {
|
|
2101
|
+
if (!deployment.isDirectory()) continue;
|
|
2102
|
+
const deploymentDirPath = join(stagingRoot, deployment.name);
|
|
2103
|
+
const journalPath = join(deploymentDirPath, ACTIVATION_JOURNAL);
|
|
2104
|
+
const fail = (component: string, error: unknown) => recordUnsettled(failures, component, error, deploymentDirPath);
|
|
2105
|
+
|
|
2106
|
+
let journal: ActivationJournal | undefined;
|
|
2107
|
+
try {
|
|
2108
|
+
journal = await readActivationJournal(journalPath);
|
|
2109
|
+
} catch (error) {
|
|
2110
|
+
// The journal itself is unreadable, so its component has to be inferred from the tree it was
|
|
2111
|
+
// going to activate. A deployment directory holding no component tree leaves only its id.
|
|
2112
|
+
const attributed = await candidateComponentName(deploymentDirPath).catch(() => undefined);
|
|
2113
|
+
await fail(attributed ?? deployment.name, error);
|
|
2114
|
+
continue;
|
|
2115
|
+
}
|
|
2116
|
+
if (!journal) {
|
|
2117
|
+
// No activation was attempted: build residue, or a candidate abandoned mid-build. The legacy
|
|
2118
|
+
// in-place recovery owns any aside it left, so there is nothing to settle.
|
|
2119
|
+
//
|
|
2120
|
+
// Removed UNDER the component's lock, and only after re-checking that no journal appeared in the
|
|
2121
|
+
// meantime. A reload cycle can run this pass while another deploy is mid-build — its candidate
|
|
2122
|
+
// has no journal yet, because the journal is written after build and validation — so an unlocked
|
|
2123
|
+
// delete here removes a live build out from under it.
|
|
2124
|
+
let owner: string | undefined;
|
|
2125
|
+
// Who to fail, if what went wrong under the lock turns out to concern an ACTIVATION rather than
|
|
2126
|
+
// this branch's opportunistic cleanup. Left unset for a sweep that could not remove a settled
|
|
2127
|
+
// directory and for a lock a live deploy is holding: neither is an unsettled activation.
|
|
2128
|
+
let activationToFail: string | undefined;
|
|
2129
|
+
// Set when the journal that appears under the lock names someone other than the sidecar owner
|
|
2130
|
+
// whose lock we took. Both names are failed, exactly as on the journaled path.
|
|
2131
|
+
let splitOwners: [string, string] | undefined;
|
|
2132
|
+
const removeResidue = async () => {
|
|
2133
|
+
// Re-read UNDER the lock, and do not swallow: the first scan raced a deploy that can publish a
|
|
2134
|
+
// journal before releasing the lock, so a journal found now must be settled rather than deleted.
|
|
2135
|
+
// Treating a read error as "no journal" would delete the evidence instead.
|
|
2136
|
+
let appeared;
|
|
2137
|
+
try {
|
|
2138
|
+
appeared = await readActivationJournal(join(deploymentDirPath, ACTIVATION_JOURNAL));
|
|
2139
|
+
} catch (error) {
|
|
2140
|
+
// A journal published between the scan's read and this one, which then cannot be READ, is an
|
|
2141
|
+
// activation whose intent is unknowable — the one thing in this branch that has to fail
|
|
2142
|
+
// closed. Attributed to the sidecar's owner, since the journal cannot name itself.
|
|
2143
|
+
activationToFail = owner ?? deployment.name;
|
|
2144
|
+
throw error;
|
|
2145
|
+
}
|
|
2146
|
+
if (appeared) {
|
|
2147
|
+
// The lock held here was taken on the SIDECAR's owner. A journal naming someone else must not
|
|
2148
|
+
// be settled under it: `settleInterruptedActivation` renames and removes that other
|
|
2149
|
+
// component's trees, whose own deploy may hold its own lock and be mid-flight. It is also the
|
|
2150
|
+
// same split evidence the journaled path fails closed for both names, so it takes the same
|
|
2151
|
+
// route rather than a second opinion here.
|
|
2152
|
+
const splitNames = splitAttributionOwners(appeared.component, owner);
|
|
2153
|
+
if (splitNames) {
|
|
2154
|
+
splitOwners = splitNames;
|
|
2155
|
+
throw splitAttributionError(deploymentDirPath, appeared.component, splitNames[0]);
|
|
2156
|
+
}
|
|
2157
|
+
activationToFail = appeared.component;
|
|
2158
|
+
await settleInterruptedActivation(componentsRootDirPath, deploymentDirPath, appeared);
|
|
2159
|
+
// Settled. Anything that fails after this — releasing the lock, say — is not this
|
|
2160
|
+
// activation's, and attributing it here would fail a correctly settled component closed.
|
|
2161
|
+
activationToFail = undefined;
|
|
2162
|
+
return;
|
|
2163
|
+
}
|
|
2164
|
+
// Re-classified UNDER the lock: the unlocked read that routed this here can predate the `.complete`
|
|
2165
|
+
// a deploy wrote before dying, and that is a retainable build, not residue.
|
|
2166
|
+
const build = await dormantBuildAt(deploymentDirPath, owner!);
|
|
2167
|
+
if (build) {
|
|
2168
|
+
catalogue(owner!, build);
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
2171
|
+
// A DESCRIBED artifact carrying a stale verdict is settled, not residue. `fail()` only ever writes
|
|
2172
|
+
// `.unsettled` beside a journal it keeps, so a marker with no journal says settlement finished
|
|
2173
|
+
// and only the marker's own removal was lost — which is exactly what a crash between a dormant
|
|
2174
|
+
// return's two unlinks leaves, and the barrier that orders them cannot run on Windows. Deleting
|
|
2175
|
+
// here would destroy a build somebody staged deliberately, whose payload may already have been
|
|
2176
|
+
// reclaimed, on the strength of a verdict that no longer applies. Clearing the marker is the
|
|
2177
|
+
// idempotent completion of the settlement that wrote it; an undescribed build stays disposable.
|
|
2178
|
+
if (await presentOrAbsent(join(deploymentDirPath, CANDIDATE_ARTIFACT_FILE))) {
|
|
2179
|
+
// A fault clearing the marker is not a licence to delete what it is attached to — and not a
|
|
2180
|
+
// licence to say nothing either. The marker survives, every worker fails the component closed
|
|
2181
|
+
// on it, and main reporting success is the split where main serves what every worker refuses.
|
|
2182
|
+
// Failing the component brings main to the workers' verdict instead, the rule the settled tail
|
|
2183
|
+
// follows, and leaves the artifact for the next pass to clear.
|
|
2184
|
+
try {
|
|
2185
|
+
await clearUnsettledVerdict(deploymentDirPath, owner!);
|
|
2186
|
+
} catch (error) {
|
|
2187
|
+
activationToFail = owner;
|
|
2188
|
+
throw error;
|
|
2189
|
+
}
|
|
2190
|
+
const settled = await dormantBuildAt(deploymentDirPath, owner!);
|
|
2191
|
+
if (settled) {
|
|
2192
|
+
logger.info?.(
|
|
2193
|
+
`Cleared a stale unsettled verdict from the staged build ${basename(deploymentDirPath)} of ` +
|
|
2194
|
+
`${owner}; its activation was already settled`
|
|
2195
|
+
);
|
|
2196
|
+
catalogue(owner!, settled);
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
// Cleared, and still not a retainable build — an incomplete or treeless staged directory. That
|
|
2200
|
+
// is residue like any other, so it falls through to the removal below.
|
|
2201
|
+
}
|
|
2202
|
+
// Cleanup, not settlement. There was no activation here — this is most often the residue a
|
|
2203
|
+
// SUCCESSFUL settlement leaves when its own sweep failed — so a sweep that fails again cannot
|
|
2204
|
+
// make anything unsettled, and recording it would refuse a live component on every worker
|
|
2205
|
+
// until the filesystem fault cleared.
|
|
2206
|
+
await rm(deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((error) =>
|
|
2207
|
+
logger.warn(`Could not clean up deploy staging ${deploymentDirPath}:`, errorForLog(error))
|
|
2208
|
+
);
|
|
2209
|
+
};
|
|
2210
|
+
// Attribution FIRST, in its own scope, and its failure is not a verdict. A journal-less directory
|
|
2211
|
+
// is what a successful settlement leaves when its best-effort sweep fails, so a `fail()` here
|
|
2212
|
+
// wrote `.unsettled` keyed by the deployment id for a deployment that never held an unsettled
|
|
2213
|
+
// activation — and nothing ever cleared it, because settlement never runs for a journal-less
|
|
2214
|
+
// directory. If that sidecar later became readable as `web`, the stale marker was attributed to
|
|
2215
|
+
// the live `web` and every worker refused it permanently. This is the same shape the deploy path
|
|
2216
|
+
// skips; the two paths now agree.
|
|
2217
|
+
try {
|
|
2218
|
+
owner = await candidateComponentName(deploymentDirPath);
|
|
2219
|
+
} catch (error) {
|
|
2220
|
+
logger.trace?.(
|
|
2221
|
+
`Leaving deploy staging ${deploymentDirPath} in place: it holds no activation journal and its ` +
|
|
2222
|
+
`ownership cannot be read: ${errorMessage(error)}`
|
|
2223
|
+
);
|
|
2224
|
+
continue;
|
|
2225
|
+
}
|
|
2226
|
+
// Catalogued WITHOUT the lock and left alone: a retained build is never removed here, so a per-directory
|
|
2227
|
+
// lock would recur on every pass and contend with sibling threads for a component nothing is deploying.
|
|
2228
|
+
if (owner) {
|
|
2229
|
+
let build: DormantBuild | undefined;
|
|
2230
|
+
try {
|
|
2231
|
+
build = await dormantBuildAt(deploymentDirPath, owner);
|
|
2232
|
+
} catch (error) {
|
|
2233
|
+
logger.warn(
|
|
2234
|
+
`Leaving deploy staging ${deploymentDirPath} in place; it could not be read:`,
|
|
2235
|
+
errorForLog(error)
|
|
2236
|
+
);
|
|
2237
|
+
continue;
|
|
2238
|
+
}
|
|
2239
|
+
if (build) {
|
|
2240
|
+
catalogue(owner, build);
|
|
2241
|
+
continue;
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
// Scoped to THIS deployment, like the journaled branch below: a lock timeout or an EIO here used to
|
|
2245
|
+
// abort the entire scan, leaving every later deployment unsettled and unmarked.
|
|
2246
|
+
try {
|
|
2247
|
+
if (owner) {
|
|
2248
|
+
await withComponentPreparationLock(join(componentsRootDirPath, owner), removeResidue, {
|
|
2249
|
+
purpose: 'activation-recovery',
|
|
2250
|
+
...RECOVERY_LOCK_WAIT,
|
|
2251
|
+
// Without this a ticket left by a CRASHED worker looks live — same pid, same process
|
|
2252
|
+
// instance — so recovery waits out the multi-hour default instead of reclaiming it.
|
|
2253
|
+
isOwnerAlive: (lockOwner) => lockOwner.pid !== process.pid || isThreadRunning(lockOwner.threadId),
|
|
2254
|
+
});
|
|
2255
|
+
} else {
|
|
2256
|
+
// NOT removed. `buildCandidateApplication` creates the deployment directory and can then spend
|
|
2257
|
+
// minutes resolving or packing before the candidate tree and its sidecar exist, so "no owner"
|
|
2258
|
+
// includes "a live build that has not got that far yet" — and deleting it races the extraction
|
|
2259
|
+
// and fails a valid deploy. Unowned residue is left for a later pass, once an owner is knowable.
|
|
2260
|
+
logger.trace?.(`Leaving unowned deploy staging ${deploymentDirPath} in place: no component names it`);
|
|
2261
|
+
}
|
|
2262
|
+
} catch (error) {
|
|
2263
|
+
// A verdict only when an activation was actually involved; a directory that cannot be swept is
|
|
2264
|
+
// not one. A lock TIMEOUT is still recorded, because it is the signal `componentLoader` uses
|
|
2265
|
+
// to defer this component until the deploy holding that lock finishes — and it leaves nothing
|
|
2266
|
+
// durable behind, since `fail()` writes no marker for a deferral.
|
|
2267
|
+
if (splitOwners) {
|
|
2268
|
+
for (const name of splitOwners) await fail(name, error);
|
|
2269
|
+
} else if (activationToFail) await fail(activationToFail, error);
|
|
2270
|
+
else if (error instanceof ComponentPreparationLockTimeoutError) await fail(owner ?? deployment.name, error);
|
|
2271
|
+
else
|
|
2272
|
+
logger.warn(
|
|
2273
|
+
`Could not settle deploy staging ${deploymentDirPath}, which holds no activation journal:`,
|
|
2274
|
+
errorForLog(error instanceof Error ? error : new Error(String(error)))
|
|
2275
|
+
);
|
|
2276
|
+
}
|
|
2277
|
+
continue;
|
|
2278
|
+
}
|
|
2279
|
+
try {
|
|
2280
|
+
// Disagreeing attributions are the one case with no automated way out: the restore gate blocks the
|
|
2281
|
+
// SIDECAR's component (it takes the union, because restoring is destructive) while settlement
|
|
2282
|
+
// keys the journal, so neither name's deploy could ever clear it. Reported as unsettleable
|
|
2283
|
+
// against the sidecar's name, which is the component actually stalled, instead of stalling it
|
|
2284
|
+
// silently on every boot.
|
|
2285
|
+
// Unreadable is not disagreement. The journal is the authority on this path — it named its
|
|
2286
|
+
// component and can settle it — so a sidecar that cannot be read must not block that; only one
|
|
2287
|
+
// that CAN be read and names something else is the wedge below.
|
|
2288
|
+
const sidecarOwner = await candidateComponentName(deploymentDirPath).catch(() => undefined);
|
|
2289
|
+
const splitNames = splitAttributionOwners(journal.component, sidecarOwner);
|
|
2290
|
+
if (splitNames) {
|
|
2291
|
+
const split = splitAttributionError(deploymentDirPath, journal.component, splitNames[0]);
|
|
2292
|
+
for (const name of splitNames) await fail(name, split);
|
|
2293
|
+
continue;
|
|
2294
|
+
}
|
|
2295
|
+
const settling = journal;
|
|
2296
|
+
await withComponentPreparationLock(
|
|
2297
|
+
join(componentsRootDirPath, settling.component),
|
|
2298
|
+
() => settleInterruptedActivation(componentsRootDirPath, deploymentDirPath, settling),
|
|
2299
|
+
{
|
|
2300
|
+
purpose: 'activation-recovery',
|
|
2301
|
+
...RECOVERY_LOCK_WAIT,
|
|
2302
|
+
isOwnerAlive: (lockOwner) => lockOwner.pid !== process.pid || isThreadRunning(lockOwner.threadId),
|
|
2303
|
+
}
|
|
2304
|
+
);
|
|
2305
|
+
} catch (error) {
|
|
2306
|
+
// The journal named its component, so attribution is exact however the settle failed.
|
|
2307
|
+
await fail(journal.component, error);
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
const maxCount = getStagingRetentionMaxCount();
|
|
2312
|
+
for (const [owner, builds] of dormant) {
|
|
2313
|
+
for (const [component, error] of await reconcileDormantBuilds(componentsRootDirPath, owner, builds, maxCount)) {
|
|
2314
|
+
if (!failures.has(component)) failures.set(component, error);
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
return failures;
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
/**
|
|
2321
|
+
* Record a settlement failure against a component. A lock TIMEOUT is a deferral, not a verdict, and only
|
|
2322
|
+
* verdicts go on disk: a held lock means a live deploy, which settles its own journal, and a marker written
|
|
2323
|
+
* here would outlive it and have `unsettleableComponentsFromDisk` fail a healthy component closed on every
|
|
2324
|
+
* worker. Everything else is written so workers reach the same verdict — a well-formed journal this pass
|
|
2325
|
+
* could not settle looks exactly like a deploy in flight otherwise. Marker write is best-effort.
|
|
2326
|
+
*/
|
|
2327
|
+
async function recordUnsettled(
|
|
2328
|
+
failures: Map<string, Error>,
|
|
2329
|
+
component: string,
|
|
2330
|
+
error: unknown,
|
|
2331
|
+
deploymentDirPath: string
|
|
2332
|
+
): Promise<void> {
|
|
2333
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
2334
|
+
if (!failures.has(component)) failures.set(component, failure);
|
|
2335
|
+
logger.error(`Could not settle the interrupted activation of ${component}:`, errorForLog(failure));
|
|
2336
|
+
if (failure instanceof ComponentPreparationLockTimeoutError) return;
|
|
2337
|
+
await writeFile(join(deploymentDirPath, UNSETTLED_MARKER), failure.message, { mode: 0o600 }).catch((markerError) =>
|
|
2338
|
+
logger.warn(`Could not record the unsettled activation of ${component}: ${errorMessage(markerError)}`)
|
|
2339
|
+
);
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
/**
|
|
2343
|
+
* Finish one owner's catalogued dormant builds after the scan. The catalog was read without the lock, so a
|
|
2344
|
+
* deploy may have published a journal into one of these directories since — and if it then died mid-swap,
|
|
2345
|
+
* only settlement brings the component back. So: settle any journal that appeared, then bound what is still
|
|
2346
|
+
* dormant. The lock is taken only when there is something to do; a lock a live deploy holds is the same
|
|
2347
|
+
* deferral the residue branch records.
|
|
2348
|
+
*/
|
|
2349
|
+
export async function reconcileDormantBuilds(
|
|
2350
|
+
componentsRootDirPath: string,
|
|
2351
|
+
owner: string,
|
|
2352
|
+
builds: DormantBuild[],
|
|
2353
|
+
maxCount: number
|
|
2354
|
+
): Promise<Map<string, Error>> {
|
|
2355
|
+
const failures = new Map<string, Error>();
|
|
2356
|
+
let journaled: DormantBuild | undefined;
|
|
2357
|
+
for (const build of builds) {
|
|
2358
|
+
// Anything but a clean ENOENT means "read it properly, under the lock".
|
|
2359
|
+
const appeared = await presentOrAbsent(join(build.deploymentDirPath, ACTIVATION_JOURNAL)).then(
|
|
2360
|
+
(stats) => stats !== undefined,
|
|
2361
|
+
() => true
|
|
2362
|
+
);
|
|
2363
|
+
if (appeared) {
|
|
2364
|
+
journaled = build;
|
|
2365
|
+
break;
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
if (!journaled && builds.length <= maxCount) return failures;
|
|
2369
|
+
try {
|
|
2370
|
+
await withComponentPreparationLock(
|
|
2371
|
+
join(componentsRootDirPath, owner),
|
|
2372
|
+
async () => {
|
|
2373
|
+
const stillDormant: DormantBuild[] = [];
|
|
2374
|
+
for (const build of builds) {
|
|
2375
|
+
let journal: ActivationJournal | undefined;
|
|
2376
|
+
try {
|
|
2377
|
+
journal = await readActivationJournal(join(build.deploymentDirPath, ACTIVATION_JOURNAL));
|
|
2378
|
+
} catch (error) {
|
|
2379
|
+
await recordUnsettled(failures, owner, error, build.deploymentDirPath);
|
|
2380
|
+
continue;
|
|
2381
|
+
}
|
|
2382
|
+
if (!journal) {
|
|
2383
|
+
stillDormant.push(build);
|
|
2384
|
+
continue;
|
|
2385
|
+
}
|
|
2386
|
+
// The lock held here is the SIDECAR owner's, as in the residue branch: a journal naming someone
|
|
2387
|
+
// else is not settled under it, and both names are failed.
|
|
2388
|
+
const splitNames = splitAttributionOwners(journal.component, owner);
|
|
2389
|
+
if (splitNames) {
|
|
2390
|
+
const split = splitAttributionError(build.deploymentDirPath, journal.component, splitNames[0]);
|
|
2391
|
+
for (const name of splitNames) await recordUnsettled(failures, name, split, build.deploymentDirPath);
|
|
2392
|
+
continue;
|
|
2393
|
+
}
|
|
2394
|
+
try {
|
|
2395
|
+
await settleInterruptedActivation(componentsRootDirPath, build.deploymentDirPath, journal);
|
|
2396
|
+
} catch (error) {
|
|
2397
|
+
await recordUnsettled(failures, journal.component, error, build.deploymentDirPath);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
if (stillDormant.length > maxCount) await pruneDormantBuilds(owner, stillDormant, maxCount);
|
|
2401
|
+
},
|
|
2402
|
+
{
|
|
2403
|
+
purpose: 'activation-recovery',
|
|
2404
|
+
...RECOVERY_LOCK_WAIT,
|
|
2405
|
+
isOwnerAlive: (lockOwner) => lockOwner.pid !== process.pid || isThreadRunning(lockOwner.threadId),
|
|
2406
|
+
}
|
|
2407
|
+
);
|
|
2408
|
+
} catch (error) {
|
|
2409
|
+
// With a journal in view this is an activation that could not be settled — recorded exactly as the
|
|
2410
|
+
// scan records one it saw directly (a timeout defers, anything else is a verdict). Without one it is
|
|
2411
|
+
// hygiene that could not run, unless a live deploy holds the lock, which defers as everywhere else.
|
|
2412
|
+
if (journaled) {
|
|
2413
|
+
await recordUnsettled(failures, owner, error, journaled.deploymentDirPath);
|
|
2414
|
+
return failures;
|
|
2415
|
+
}
|
|
2416
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
2417
|
+
if (failure instanceof ComponentPreparationLockTimeoutError) {
|
|
2418
|
+
if (!failures.has(owner)) failures.set(owner, failure);
|
|
2419
|
+
logger.info?.(`Deferred pruning the dormant staged builds of ${owner}: a deploy holds its lock`);
|
|
2420
|
+
} else {
|
|
2421
|
+
logger.warn(`Could not prune the dormant staged builds of ${owner}:`, errorForLog(failure));
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
return failures;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
/**
|
|
2428
|
+
* Retire and sweep the rollback records a settled activation leaves. Retiring throws — it is what makes a
|
|
2429
|
+
* record non-authoritative, so a caller that removed the journal without it re-creates the inversion the
|
|
2430
|
+
* journal prevents. Sweeping the displaced tree only costs disk, so it is logged.
|
|
2431
|
+
*/
|
|
2432
|
+
async function sweepAsideRecords(
|
|
2433
|
+
records: string[],
|
|
2434
|
+
componentName: string,
|
|
2435
|
+
liveDirPath: string,
|
|
2436
|
+
asideStagingDir: string
|
|
2437
|
+
): Promise<void> {
|
|
2438
|
+
for (const record of records) {
|
|
2439
|
+
// RETIRING IS CORRECTNESS, not hygiene: the retired marker is what stops the legacy pass treating this
|
|
2440
|
+
// record as authoritative and restoring the displaced tree over the candidate that was just rolled
|
|
2441
|
+
// forward, once the journal that would otherwise hold it back is gone. A record left un-retired while the journal is removed re-creates exactly
|
|
2442
|
+
// the inversion this protocol exists to prevent, so a failure here PROPAGATES — the caller keeps the
|
|
2443
|
+
// journal and the next start retries.
|
|
2444
|
+
const retiredMarkerPath = await retireExtractionAside(record);
|
|
2445
|
+
// Sweeping the displaced tree is hygiene: it bounds disk, and a failure costs space rather than
|
|
2446
|
+
// correctness, so it is logged. The retired marker above already makes the record non-authoritative.
|
|
2447
|
+
await cleanupExtractionPaths(
|
|
2448
|
+
{ name: componentName, dirPath: liveDirPath, logger },
|
|
2449
|
+
asideStagingDir,
|
|
2450
|
+
new Set([record, retiredMarkerPath])
|
|
2451
|
+
).catch((error) => logger.warn(`Settled ${componentName} but could not sweep ${record}:`, errorForLog(error)));
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/**
|
|
2456
|
+
* Clear an earlier failed recovery's verdict once this settlement has decided. Treated as CORRECTNESS, not
|
|
2457
|
+
* cleanup: main would report the component settled and load it while every worker read the stale marker and
|
|
2458
|
+
* failed it closed, so a failure here throws and lets main reach the same verdict. The journal outlives it
|
|
2459
|
+
* either way, so the next start settles again.
|
|
2460
|
+
*/
|
|
2461
|
+
async function clearUnsettledVerdict(deploymentDirPath: string, componentName: string): Promise<void> {
|
|
2462
|
+
try {
|
|
2463
|
+
await rm(join(deploymentDirPath, UNSETTLED_MARKER), { force: true });
|
|
2464
|
+
// Flushed here, not with whatever follows: the journal's removal must never be the one that survives
|
|
2465
|
+
// a crash alone, or the verdict outlives the only thing that would settle it again.
|
|
2466
|
+
await syncDirectory(deploymentDirPath);
|
|
2467
|
+
} catch (error) {
|
|
2468
|
+
throw new Error(
|
|
2469
|
+
`Could not clear the stale unsettled marker of ${componentName} at ` +
|
|
2470
|
+
`${join(deploymentDirPath, UNSETTLED_MARKER)}; the component stays failed closed on every thread ` +
|
|
2471
|
+
`until that file can be removed: ${errorMessage(error)}`,
|
|
2472
|
+
{ cause: error }
|
|
2473
|
+
);
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
/**
|
|
2478
|
+
* One interrupted activation, under the component preparation lock. Ambiguity exists only while the live
|
|
2479
|
+
* path is absent, and there the `complete` marker is the roll-forward authority: without it the candidate
|
|
2480
|
+
* was never validated, so the committed tree in the aside wins. Every branch is idempotent, so a crash
|
|
2481
|
+
* during recovery is settled by the next run.
|
|
2482
|
+
*/
|
|
2483
|
+
async function settleInterruptedActivation(
|
|
2484
|
+
componentsRootDirPath: string,
|
|
2485
|
+
deploymentDirPath: string,
|
|
2486
|
+
journal: ActivationJournal
|
|
2487
|
+
): Promise<void> {
|
|
2488
|
+
const liveDirPath = join(componentsRootDirPath, journal.component);
|
|
2489
|
+
const candidateDirPath = join(deploymentDirPath, journal.component);
|
|
2490
|
+
const asideStagingDir = extractionStagingDirectory(liveDirPath);
|
|
2491
|
+
const journalPath = join(deploymentDirPath, ACTIVATION_JOURNAL);
|
|
2492
|
+
|
|
2493
|
+
const exists = async (path: string) =>
|
|
2494
|
+
lstat(path).then(
|
|
2495
|
+
() => true,
|
|
2496
|
+
(error) => {
|
|
2497
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
|
2498
|
+
throw error;
|
|
2499
|
+
}
|
|
2500
|
+
);
|
|
2501
|
+
const liveExists = await exists(liveDirPath);
|
|
2502
|
+
const candidateExists = await exists(candidateDirPath);
|
|
2503
|
+
const candidateComplete = await exists(join(deploymentDirPath, CANDIDATE_COMPLETE_MARKER));
|
|
2504
|
+
const asideRecords = await inProgressAsideRecords(asideStagingDir);
|
|
2505
|
+
|
|
2506
|
+
const rollForward = async () => {
|
|
2507
|
+
if (!liveExists) await renameThroughTransientHolder(candidateDirPath, liveDirPath);
|
|
2508
|
+
// Unconditional, not only when THIS pass performed the rename: a crash after normal activation
|
|
2509
|
+
// renamed the candidate but before it repaired the links leaves live present with stale targets, and
|
|
2510
|
+
// gating the repair on the rename would skip exactly that case. Idempotent when there is nothing to
|
|
2511
|
+
// re-point.
|
|
2512
|
+
await repairRelocatedDependencyLinks(liveDirPath, candidateDirPath);
|
|
2513
|
+
await syncRenameParents(candidateDirPath, liveDirPath);
|
|
2514
|
+
// Retiring PROPAGATES from here: the retired marker is what stops the legacy pass restoring the tree
|
|
2515
|
+
// this roll-forward just displaced. Failing the component closed and retrying at the next start is
|
|
2516
|
+
// the cheaper mistake — the journal survives, so the verdict is re-derivable. Only the disk sweep
|
|
2517
|
+
// inside is best-effort.
|
|
2518
|
+
await sweepAsideRecords(asideRecords, journal.component, liveDirPath, asideStagingDir);
|
|
2519
|
+
};
|
|
2520
|
+
const rollBack = async (restoreFrom?: string) => {
|
|
2521
|
+
if (restoreFrom) {
|
|
2522
|
+
await renameThroughTransientHolder(restoreFrom, liveDirPath);
|
|
2523
|
+
await syncRenameParents(restoreFrom, liveDirPath);
|
|
2524
|
+
}
|
|
2525
|
+
for (const record of asideRecords) {
|
|
2526
|
+
try {
|
|
2527
|
+
await rm(record, { recursive: true, force: true });
|
|
2528
|
+
} catch (error) {
|
|
2529
|
+
// A record that survives removal is still AUTHORITATIVE to the legacy pass, and settlement is
|
|
2530
|
+
// about to remove the journal that holds that pass back — so it would restore this older tree
|
|
2531
|
+
// over the component just rolled back. Retiring is what makes a record non-authoritative, so
|
|
2532
|
+
// do that instead of logging and moving on. A retire that ALSO fails propagates: the journal
|
|
2533
|
+
// then survives and the next start retries, which is the same contract roll-forward uses.
|
|
2534
|
+
logger.warn(`Rolled ${journal.component} back but could not remove ${record}:`, errorForLog(error));
|
|
2535
|
+
await retireExtractionAside(record);
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
};
|
|
2539
|
+
|
|
2540
|
+
if (!liveExists) {
|
|
2541
|
+
if (candidateExists && candidateComplete) {
|
|
2542
|
+
await rollForward();
|
|
2543
|
+
} else {
|
|
2544
|
+
// The tree that was moved aside is the last committed one. A `-prior-absent` record means there
|
|
2545
|
+
// was nothing live to begin with, so rolling back means leaving the component absent.
|
|
2546
|
+
const restorable = asideRecords.find((record) => !record.endsWith(PRIOR_ABSENT_RECORD_SUFFIX));
|
|
2547
|
+
if (!restorable && !asideRecords.length) {
|
|
2548
|
+
throw new Error(
|
|
2549
|
+
`Cannot settle the interrupted activation of ${journal.component}: it has neither a live tree, ` +
|
|
2550
|
+
`a complete candidate, nor a rollback record, so no version of it can be recovered`
|
|
2551
|
+
);
|
|
2552
|
+
}
|
|
2553
|
+
await rollBack(restorable);
|
|
2554
|
+
}
|
|
2555
|
+
} else if (candidateExists) {
|
|
2556
|
+
// Live and candidate both present normally means B1 never ran: the swap had not started, so the live
|
|
2557
|
+
// tree stands and the candidate goes.
|
|
2558
|
+
//
|
|
2559
|
+
// Unless a rollback record says B1 DID run. Then the committed tree is the one in the aside, and
|
|
2560
|
+
// whatever sits at the live path was put there afterwards — a previous-version worker recreating its
|
|
2561
|
+
// own directory, the case the extraction path guards with `identifyRollbackPlaceholder`. Rolling back
|
|
2562
|
+
// there deletes the committed tree AND the validated candidate and leaves that stub serving, so this
|
|
2563
|
+
// fails closed instead: both trees stay on disk for an operator to choose between.
|
|
2564
|
+
// NOT conditioned on the candidate being complete. `rollBack()` below removes every aside record, and
|
|
2565
|
+
// with a record present that tree is the last committed one — so deleting it destroys the only
|
|
2566
|
+
// surviving copy of the previous release whether or not the candidate was ever validated. What
|
|
2567
|
+
// `.complete` decides is which tree we would prefer, not whether discarding the other is safe.
|
|
2568
|
+
const displaced = asideRecords.find((record) => !record.endsWith(PRIOR_ABSENT_RECORD_SUFFIX));
|
|
2569
|
+
if (displaced) {
|
|
2570
|
+
throw new Error(
|
|
2571
|
+
`Cannot settle the interrupted activation of ${journal.component}: its previous tree was moved to ` +
|
|
2572
|
+
`${displaced}, but ${liveDirPath} exists again — something recreated it after the deploy moved ` +
|
|
2573
|
+
`it aside, so which tree is current cannot be determined without losing one of them. Remove ` +
|
|
2574
|
+
`whichever of the two is not the release you want once you have determined which that is.`
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2577
|
+
// A STAGED artifact is not a disposable build. `rollBack()` removes the whole deployment directory,
|
|
2578
|
+
// which is right for an immediate deploy — the candidate came from a payload the operator still has —
|
|
2579
|
+
// but wrong for one somebody staged deliberately and may have had its payload reclaimed. The
|
|
2580
|
+
// descriptor is what tells the two apart, and it is on disk precisely so recovery can. Returning the
|
|
2581
|
+
// artifact to dormant by removing only the journal leaves it exactly as `deployment_id` expects it.
|
|
2582
|
+
if (await presentOrAbsent(join(deploymentDirPath, CANDIDATE_ARTIFACT_FILE))) {
|
|
2583
|
+
// This branch returns early and so reaches none of the settled tail below — which can be careless
|
|
2584
|
+
// about both, because it removes the whole directory afterwards. This one keeps it, so the order
|
|
2585
|
+
// the two unlinks REACH STORAGE decides whether the artifact survives: `.unsettled` with no
|
|
2586
|
+
// journal is a verdict nothing will ever settle and the next retention pass deletes the build on.
|
|
2587
|
+
// Its flush is the barrier between the two unlinks: skipping it would let the journal's removal
|
|
2588
|
+
// persist alone, leaving a verdict nothing will settle again. Throwing is safe here and leaves the
|
|
2589
|
+
// journal, so the next start settles again. Windows cannot fsync a directory, which is why the
|
|
2590
|
+
// residue pass also refuses to read this state as disposable — see DESIGN.md.
|
|
2591
|
+
await clearUnsettledVerdict(deploymentDirPath, journal.component);
|
|
2592
|
+
await rm(journalPath, { force: true });
|
|
2593
|
+
// Nothing may throw past the journal removal, the rule the settled tail follows: the caller
|
|
2594
|
+
// records a failure by writing `.unsettled`, which is the state this branch exists to avoid. An
|
|
2595
|
+
// unflushed removal is the safe direction — a power loss resurrects a journal that settles again.
|
|
2596
|
+
await syncDirectory(deploymentDirPath).catch((error) =>
|
|
2597
|
+
logger.warn(
|
|
2598
|
+
`Returned the staged build ${basename(deploymentDirPath)} of ${journal.component} to dormant but ` +
|
|
2599
|
+
`could not flush that to storage; a power loss could resurrect its activation journal:`,
|
|
2600
|
+
errorForLog(error)
|
|
2601
|
+
)
|
|
2602
|
+
);
|
|
2603
|
+
logger.info?.(
|
|
2604
|
+
`Returned the staged build ${basename(deploymentDirPath)} of ${journal.component} to dormant after an ` +
|
|
2605
|
+
`activation that never moved its live tree aside`
|
|
2606
|
+
);
|
|
2607
|
+
return;
|
|
2608
|
+
}
|
|
2609
|
+
await rollBack();
|
|
2610
|
+
} else {
|
|
2611
|
+
// The candidate is already live; only the tail of the transaction was lost.
|
|
2612
|
+
await rollForward();
|
|
2613
|
+
}
|
|
2614
|
+
|
|
2615
|
+
// The same ordering barrier normal activation uses, and for the same reason: the journal is the only
|
|
2616
|
+
// thing left telling the legacy pass not to restore an aside. Removing it while an
|
|
2617
|
+
// `.in-progress-*` record still names the displaced tree — because the retire or the sweep did not
|
|
2618
|
+
// reach storage — lets that pass put the old version back over the new one at the next start. Flush the
|
|
2619
|
+
// aside directory first, and leave the journal in place if that cannot be confirmed; recovery is
|
|
2620
|
+
// idempotent, so the next run settles it again.
|
|
2621
|
+
try {
|
|
2622
|
+
await syncDirectory(asideStagingDir);
|
|
2623
|
+
await syncDirectory(dirname(asideStagingDir));
|
|
2624
|
+
} catch (error) {
|
|
2625
|
+
logger.warn(
|
|
2626
|
+
`Settled the interrupted activation of ${journal.component} but could not flush its rollback record; ` +
|
|
2627
|
+
`leaving the journal for the next start: ${errorMessage(error)}`
|
|
2628
|
+
);
|
|
2629
|
+
return;
|
|
2630
|
+
}
|
|
2631
|
+
// Best-effort, matching the activation path: the activation is settled by this point, so a transient
|
|
2632
|
+
// EBUSY removing staging must not throw out of the recovery pass and take the other components with it.
|
|
2633
|
+
// An earlier failed recovery may have left an unsettled marker here. Cleared BEFORE the journal and
|
|
2634
|
+
// treated as correctness: main would report this component settled and load it, while every worker read
|
|
2635
|
+
// the stale marker and failed it closed.
|
|
2636
|
+
await clearUnsettledVerdict(deploymentDirPath, journal.component);
|
|
2637
|
+
await rm(journalPath, { force: true }).catch((error) =>
|
|
2638
|
+
logger.warn(`Settled ${journal.component} but could not remove its activation journal:`, errorForLog(error))
|
|
2639
|
+
);
|
|
2640
|
+
await rm(deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((error) =>
|
|
2641
|
+
logger.warn(`Settled ${journal.component} but could not clean up its staging directory:`, errorForLog(error))
|
|
2642
|
+
);
|
|
2643
|
+
await rmdir(dirname(deploymentDirPath)).catch(() => {});
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
/** Mark a candidate build+validation complete. Idempotent, so a retried activation is not a failure. */
|
|
2647
|
+
/**
|
|
2648
|
+
* fsync the candidate's contents before `.complete` vouches for them — otherwise the control files can
|
|
2649
|
+
* outlive the tree after a power loss and recovery rolls forward onto a truncated one.
|
|
2650
|
+
*/
|
|
2651
|
+
// Codes that mean "this platform or filesystem will not fsync this handle", as opposed to "the write did
|
|
2652
|
+
// not reach storage". Windows raises EPERM fsyncing perfectly healthy files, and network/overlay mounts
|
|
2653
|
+
// return EINVAL or ENOTSUP — none of which say anything about durability, and all of which would otherwise
|
|
2654
|
+
// fail every deploy on those platforms.
|
|
2655
|
+
const UNSUPPORTED_SYNC_CODES = new Set(['EPERM', 'EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'EBADF', 'EISDIR']);
|
|
2656
|
+
|
|
2657
|
+
function isUnsupportedSync(error: unknown): boolean {
|
|
2658
|
+
return UNSUPPORTED_SYNC_CODES.has((error as NodeJS.ErrnoException)?.code ?? '');
|
|
2659
|
+
}
|
|
2660
|
+
|
|
2661
|
+
// How many file syncs run at once while flushing a candidate. Serial open/sync/close over a large
|
|
2662
|
+
// dependency tree adds seconds to every activation, all of it under the component preparation lock; a small
|
|
2663
|
+
// fan-out keeps the ordering guarantee (everything is synced before `.complete` is written) without paying
|
|
2664
|
+
// per-file latency one file at a time.
|
|
2665
|
+
const CANDIDATE_SYNC_CONCURRENCY = 16;
|
|
2666
|
+
// Directories walked at once when re-pointing dependency links after a swap; a pnpm or monorepo tree is
|
|
2667
|
+
// thousands of directories and a serial depth-first walk after every activation is a real cost.
|
|
2668
|
+
const LINK_REPAIR_CONCURRENCY = 8;
|
|
2669
|
+
|
|
2670
|
+
async function syncTreeContents(rootPath: string, foreignTree = false): Promise<void> {
|
|
2671
|
+
// Real durability failures propagate: the deploy fails, which is safe because the live tree is
|
|
2672
|
+
// untouched. Platform "cannot fsync this handle" codes do not — treating those as durability failures
|
|
2673
|
+
// fails every deploy on Windows.
|
|
2674
|
+
const entries = await readdir(rootPath, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
|
|
2675
|
+
// Same reasoning as the per-file tolerance below: a directory inside a foreign tree that this uid
|
|
2676
|
+
// cannot list is not ours to make durable, and failing here fails a deploy over a directory the
|
|
2677
|
+
// deploy never wrote.
|
|
2678
|
+
if (foreignTree && error?.code === 'EACCES') {
|
|
2679
|
+
logger.trace?.(`Sync of ${rootPath} unavailable: ${errorMessage(error)}`);
|
|
2680
|
+
return undefined;
|
|
2681
|
+
}
|
|
2682
|
+
throw error;
|
|
2683
|
+
});
|
|
2684
|
+
if (!entries) return;
|
|
2685
|
+
const syncFile = async (entryPath: string) => {
|
|
2686
|
+
let handle;
|
|
2687
|
+
try {
|
|
2688
|
+
handle = await open(entryPath, 'r');
|
|
2689
|
+
} catch (error) {
|
|
2690
|
+
// `foreignTree`: a `file:<directory>` candidate is a symlink to a tree this deploy does not own,
|
|
2691
|
+
// so it can hold files the Harper uid cannot open. Those are not ours to make durable and their
|
|
2692
|
+
// EACCES says nothing about whether the install output beside them reached storage — while
|
|
2693
|
+
// failing here would fail an otherwise valid deploy over a file the deploy never touched. The
|
|
2694
|
+
// install output itself is written as this uid, so it is readable and still fsynced.
|
|
2695
|
+
//
|
|
2696
|
+
// The limit of that: this cannot tell a developer's unreadable source file from an install
|
|
2697
|
+
// script that deliberately made its OWN output unreadable, so `.complete` could certify output
|
|
2698
|
+
// that was never synced. Only for a `file:` candidate, and only for a script that chmods its
|
|
2699
|
+
// own artifacts away from the uid that has to run them.
|
|
2700
|
+
if (isUnsupportedSync(error) || (foreignTree && (error as NodeJS.ErrnoException)?.code === 'EACCES')) {
|
|
2701
|
+
logger.trace?.(`Sync of ${entryPath} unavailable: ${errorMessage(error)}`);
|
|
2702
|
+
return;
|
|
2703
|
+
}
|
|
2704
|
+
throw error;
|
|
2705
|
+
}
|
|
2706
|
+
try {
|
|
2707
|
+
await handle.sync();
|
|
2708
|
+
} catch (error) {
|
|
2709
|
+
if (!isUnsupportedSync(error)) throw error;
|
|
2710
|
+
logger.trace?.(`Sync of ${entryPath} unsupported: ${errorMessage(error)}`);
|
|
2711
|
+
} finally {
|
|
2712
|
+
await handle.close().catch(() => {});
|
|
2713
|
+
}
|
|
2714
|
+
};
|
|
2715
|
+
const pending: Promise<void>[] = [];
|
|
2716
|
+
for (const entry of entries) {
|
|
2717
|
+
const entryPath = join(rootPath, entry.name);
|
|
2718
|
+
if (entry.isDirectory()) {
|
|
2719
|
+
await syncTreeContents(entryPath, foreignTree);
|
|
2720
|
+
} else if (entry.isFile()) {
|
|
2721
|
+
pending.push(syncFile(entryPath));
|
|
2722
|
+
if (pending.length >= CANDIDATE_SYNC_CONCURRENCY) {
|
|
2723
|
+
await Promise.all(pending.splice(0));
|
|
2724
|
+
}
|
|
2725
|
+
}
|
|
2726
|
+
}
|
|
2727
|
+
await Promise.all(pending);
|
|
2728
|
+
await syncDirectory(rootPath);
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
export async function markCandidateComplete(
|
|
2732
|
+
componentDirPath: string,
|
|
2733
|
+
deploymentId: string,
|
|
2734
|
+
componentName: string
|
|
2735
|
+
): Promise<void> {
|
|
2736
|
+
// Contents first: `.complete` is roll-forward AUTHORITY, so it must not be durable before the tree it
|
|
2737
|
+
// vouches for.
|
|
2738
|
+
//
|
|
2739
|
+
// A `file:<directory>` candidate IS a symlink to a tree this deploy does not own, but the dependency
|
|
2740
|
+
// install writes THROUGH it — so the tree still has to be walked, or the install output `.complete`
|
|
2741
|
+
// vouches for is never made durable. Only the foreign files alongside it are tolerated: see
|
|
2742
|
+
// `syncTreeContents`.
|
|
2743
|
+
const candidatePath = candidateApplicationPath(componentDirPath, deploymentId);
|
|
2744
|
+
const candidateIsLink = await lstat(candidatePath).then(
|
|
2745
|
+
(stats) => stats.isSymbolicLink(),
|
|
2746
|
+
(error) => {
|
|
2747
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
|
2748
|
+
throw error;
|
|
2749
|
+
}
|
|
2750
|
+
);
|
|
2751
|
+
await syncTreeContents(candidatePath, candidateIsLink);
|
|
2752
|
+
try {
|
|
2753
|
+
await writeControlFileDurably(candidateComponentFilePath(componentDirPath, deploymentId), componentName);
|
|
2754
|
+
} catch (error) {
|
|
2755
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
2756
|
+
}
|
|
2757
|
+
try {
|
|
2758
|
+
await writeControlFileDurably(candidateCompleteMarkerPath(componentDirPath, deploymentId), '');
|
|
2759
|
+
} catch (error) {
|
|
2760
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
2761
|
+
}
|
|
2762
|
+
}
|
|
2763
|
+
|
|
2764
|
+
/**
|
|
2765
|
+
* Make the newly created ancestors of a deployment directory durable, child-first.
|
|
2766
|
+
*
|
|
2767
|
+
* Only a staged artifact needs this. A deploy's own candidate is transient — power loss just abandons a
|
|
2768
|
+
* build nobody was told about — but a stage is ACKNOWLEDGED, and `writeControlFileDurably` flushes only the
|
|
2769
|
+
* control file's immediate parent while `ensureSecureStagingDirectory` flushes none. Without this a stage
|
|
2770
|
+
* can report success before the `.deploy-staging/<id>` entry exists on storage, and the automatic payload
|
|
2771
|
+
* reclaim may already have dropped the tarball it could have been rebuilt from. Best-effort on Windows, like
|
|
2772
|
+
* every other directory sync here.
|
|
2773
|
+
*/
|
|
2774
|
+
async function syncArtifactAncestors(deploymentDirPath: string): Promise<void> {
|
|
2775
|
+
const stagingRoot = dirname(deploymentDirPath);
|
|
2776
|
+
for (const directory of [deploymentDirPath, stagingRoot, dirname(stagingRoot)]) {
|
|
2777
|
+
await syncDirectory(directory);
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
|
|
2781
|
+
/**
|
|
2782
|
+
* Make a built and validated candidate live, as one compensating transaction over two effects: the live tree
|
|
2783
|
+
* moves aside, then the candidate takes its place. Root config is NOT one of them — for an immediate deploy
|
|
2784
|
+
* it is still published before the build, unchanged, and making it transactional is tracked separately
|
|
2785
|
+
* (#2315). A delayed activation hands its artifact's recorded entry in as `afterJournal`, which publishes it
|
|
2786
|
+
* from inside the window a crash rolls forward from — see that call site.
|
|
2787
|
+
*
|
|
2788
|
+
* The candidate must ALREADY be certified: `markCandidateComplete` is the caller's, so a delayed activation
|
|
2789
|
+
* does not re-walk and re-fsync a whole dependency tree it certified when it was built. The activation
|
|
2790
|
+
* journal is still written and fsynced BEFORE the first rename, so a crash anywhere below is recoverable —
|
|
2791
|
+
* see `settleInterruptedActivation` for the state matrix. The second rename is the COMMIT POINT: nothing
|
|
2792
|
+
* after it may compensate, because the live path holds the candidate and renaming the aside back over it
|
|
2793
|
+
* cannot succeed.
|
|
2794
|
+
*/
|
|
2795
|
+
export async function activateCandidateApplication(
|
|
2796
|
+
application: Application,
|
|
2797
|
+
deploymentId: string,
|
|
2798
|
+
options: { afterJournal?: () => Promise<(() => Promise<void>) | void> } = {}
|
|
2799
|
+
): Promise<void> {
|
|
2800
|
+
const liveDirPath = application.dirPath;
|
|
2801
|
+
const candidateDirPath = candidateApplicationPath(liveDirPath, deploymentId);
|
|
2802
|
+
const deploymentDirPath = candidateDeploymentDirPath(liveDirPath, deploymentId);
|
|
2803
|
+
const asideStagingDir = extractionStagingDirectory(liveDirPath);
|
|
2804
|
+
|
|
2805
|
+
// A symlink counts: a `file:<directory>` deploy links the source rather than extracting it, and that
|
|
2806
|
+
// link is what gets swapped into the live path.
|
|
2807
|
+
const candidateStat = await lstat(candidateDirPath).catch((error) => {
|
|
2808
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
|
2809
|
+
throw error;
|
|
2810
|
+
});
|
|
2811
|
+
if (!candidateStat || !(candidateStat.isDirectory() || candidateStat.isSymbolicLink())) {
|
|
2812
|
+
throw new Error(`Cannot activate ${application.name}: no candidate build at ${candidateDirPath}`);
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
const journalPath = activationJournalPath(liveDirPath, deploymentId);
|
|
2816
|
+
|
|
2817
|
+
// B1 — the live tree moves aside. It stays the rollback source until B4 retires it.
|
|
2818
|
+
let asidePath: string | undefined;
|
|
2819
|
+
let priorAbsentRecordPath: string | undefined;
|
|
2820
|
+
// The swap below moves the previous tree back and forth around every wait, so a chosen aside path no
|
|
2821
|
+
// longer implies the tree is at it — and compensation needs to know which.
|
|
2822
|
+
let liveIsDisplaced = false;
|
|
2823
|
+
const restoreLive = async () => {
|
|
2824
|
+
if (liveIsDisplaced) {
|
|
2825
|
+
await renameThroughTransientHolder(asidePath!, liveDirPath);
|
|
2826
|
+
liveIsDisplaced = false;
|
|
2827
|
+
} else if (priorAbsentRecordPath) await rm(priorAbsentRecordPath, { force: true });
|
|
2828
|
+
// Nothing was displaced and no record was written, so there is nothing to put back and no rename to
|
|
2829
|
+
// flush — the failure happened before the first effect.
|
|
2830
|
+
if (asidePath || priorAbsentRecordPath) {
|
|
2831
|
+
await syncRenameParents(asidePath ?? priorAbsentRecordPath!, liveDirPath);
|
|
2832
|
+
}
|
|
2833
|
+
};
|
|
2834
|
+
|
|
2835
|
+
/**
|
|
2836
|
+
* Put a compensated candidate back to DORMANT — complete, described, no journal — so it is a retryable
|
|
2837
|
+
* artifact rather than one the next preparation destroys.
|
|
2838
|
+
*
|
|
2839
|
+
* Without this, a failure that leaves the journal in place makes `settleStagingForComponent` read
|
|
2840
|
+
* live-plus-candidate as an activation that never got there and remove the whole deployment directory;
|
|
2841
|
+
* for a first deploy the restored state is live-ABSENT, and it rolls the candidate forward instead,
|
|
2842
|
+
* ahead of the caller's own verification. Both destroy an artifact whose whole purpose is to be
|
|
2843
|
+
* activated again.
|
|
2844
|
+
*
|
|
2845
|
+
* The unlink needs its own barrier: syncing the aside directories persists the rollback record's
|
|
2846
|
+
* disposal, not the journal's, so without this sync a power loss resurrects a journal the operator was
|
|
2847
|
+
* told had been rolled back. Best-effort by necessity — failing here must not replace the activation
|
|
2848
|
+
* failure the caller is reporting — so a failure says explicitly that the artifact is not retryable, and
|
|
2849
|
+
* the surviving journal is exactly what startup recovery settles.
|
|
2850
|
+
*/
|
|
2851
|
+
const returnToDormant = async () => {
|
|
2852
|
+
try {
|
|
2853
|
+
await rm(journalPath, { force: true });
|
|
2854
|
+
} catch (error) {
|
|
2855
|
+
application.logger.warn(
|
|
2856
|
+
`Restored ${application.name} after a failed activation, but its staged build ${deploymentId} still ` +
|
|
2857
|
+
`carries an activation journal and is not retryable until recovery settles it:`,
|
|
2858
|
+
error
|
|
2859
|
+
);
|
|
2860
|
+
return;
|
|
2861
|
+
}
|
|
2862
|
+
// The journal is gone in this process, so the artifact is retryable now; what is uncertain is whether
|
|
2863
|
+
// its removal reached storage.
|
|
2864
|
+
await syncDirectory(deploymentDirPath).catch((error) =>
|
|
2865
|
+
application.logger.warn(
|
|
2866
|
+
`Restored ${application.name} and returned its staged build ${deploymentId} to a retryable state, but ` +
|
|
2867
|
+
`could not flush that to storage; a power loss could resurrect its activation journal:`,
|
|
2868
|
+
error
|
|
2869
|
+
)
|
|
2870
|
+
);
|
|
2871
|
+
};
|
|
2872
|
+
|
|
2873
|
+
/**
|
|
2874
|
+
* One pre-commit failure boundary, with the journal write inside it: every step from there to the commit
|
|
2875
|
+
* rename leaves a journal behind if it only rethrows, and the next settlement then deletes a certified
|
|
2876
|
+
* artifact, or (first deploy, live absent) activates it with nobody asking. Nothing below the commit
|
|
2877
|
+
* rename may enter this catch — see B2.
|
|
2878
|
+
*/
|
|
2879
|
+
let pendingEffect = 'record the activation';
|
|
2880
|
+
let undoAfterJournal: (() => Promise<void>) | void;
|
|
2881
|
+
try {
|
|
2882
|
+
try {
|
|
2883
|
+
await writeControlFileDurably(
|
|
2884
|
+
journalPath,
|
|
2885
|
+
JSON.stringify({
|
|
2886
|
+
v: ACTIVATION_JOURNAL_VERSION,
|
|
2887
|
+
component: application.name,
|
|
2888
|
+
candidateId: deploymentId,
|
|
2889
|
+
})
|
|
2890
|
+
);
|
|
2891
|
+
} catch (error) {
|
|
2892
|
+
// An existing journal is a retry of this same activation, not a conflict.
|
|
2893
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
2894
|
+
}
|
|
2895
|
+
|
|
2896
|
+
pendingEffect = 'prepare the component staging directory';
|
|
2897
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
2898
|
+
pendingEffect = 'read the live component directory';
|
|
2899
|
+
const liveExists = await lstat(liveDirPath).then(
|
|
2900
|
+
() => true,
|
|
2901
|
+
(error) => {
|
|
2902
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
|
2903
|
+
throw error;
|
|
2904
|
+
}
|
|
2905
|
+
);
|
|
2906
|
+
application.isNewComponent = !liveExists;
|
|
2907
|
+
|
|
2908
|
+
pendingEffect = 'move the previous version aside';
|
|
2909
|
+
if (liveExists) {
|
|
2910
|
+
asidePath = join(asideStagingDir, `${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}`);
|
|
2911
|
+
await renameThroughTransientHolder(liveDirPath, asidePath);
|
|
2912
|
+
liveIsDisplaced = true;
|
|
2913
|
+
} else {
|
|
2914
|
+
priorAbsentRecordPath = join(
|
|
2915
|
+
asideStagingDir,
|
|
2916
|
+
`${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}${PRIOR_ABSENT_RECORD_SUFFIX}`
|
|
2917
|
+
);
|
|
2918
|
+
await writeFile(priorAbsentRecordPath, '', { flag: 'wx', mode: 0o600 });
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
// Letting a storage failure escape here leaves live already moved aside, and the caller reads an
|
|
2922
|
+
// uncompensated throw as an ordinary build failure and discards the candidate, its `.complete` marker
|
|
2923
|
+
// and its journal: the component ends up with no version at all and nothing saying how to get one back.
|
|
2924
|
+
pendingEffect = 'record the displaced component directory';
|
|
2925
|
+
await syncRenameParents(liveDirPath, asidePath ?? priorAbsentRecordPath!);
|
|
2926
|
+
|
|
2927
|
+
// Config is published HERE — after B1, before the commit — because this is the only point where the
|
|
2928
|
+
// on-disk state recovery would find rolls FORWARD to the certified artifact: live is displaced, the
|
|
2929
|
+
// candidate is complete, and the rollback record exists. Publishing before B1 (with or without the
|
|
2930
|
+
// journal) leaves live present and the candidate present with no rollback record, which settlement
|
|
2931
|
+
// reads as an activation that never started: it deletes the deployment directory, and the next boot
|
|
2932
|
+
// re-resolves the published package identifier from the registry instead — the substitution this step
|
|
2933
|
+
// exists to prevent.
|
|
2934
|
+
//
|
|
2935
|
+
// A crash in the remaining window — after the roll-forward state exists but before this publish — is
|
|
2936
|
+
// the inverse hazard: `rollForward()` renames the candidate live and publishes nothing, so the
|
|
2937
|
+
// certified artifact serves under the PREVIOUS release's config. That includes its ISOLATION intent,
|
|
2938
|
+
// which is a containment boundary and not just a version string: a component staged to run isolated
|
|
2939
|
+
// comes back non-isolated after an ordinary crash, with nothing in the operation reporting it.
|
|
2940
|
+
// Closing it needs config to be an effect of the journal itself, which is #2315 step 3.
|
|
2941
|
+
pendingEffect = 'publish the root configuration';
|
|
2942
|
+
undoAfterJournal = await options.afterJournal?.();
|
|
2943
|
+
|
|
2944
|
+
// B2 — the candidate becomes live. THE RENAME IS THE COMMIT POINT: nothing after it may compensate,
|
|
2945
|
+
// because the live path now holds the candidate and renaming the aside back over it cannot succeed. A
|
|
2946
|
+
// compensating step there fails its own rollback and reports a failure for a deploy that is live. It is
|
|
2947
|
+
// the LAST statement in this block for that reason.
|
|
2948
|
+
pendingEffect = 'move the candidate into place';
|
|
2949
|
+
await renameThroughTransientHolder(candidateDirPath, liveDirPath, {
|
|
2950
|
+
// The previous version occupies the live path through the wait rather than the component being
|
|
2951
|
+
// absent for the whole budget: a read of a component file, and any concurrent scan of the
|
|
2952
|
+
// components root, still finds the last committed tree. (Watchers are already paused for the
|
|
2953
|
+
// deploy, so they are not what this protects.) A first-ever deploy has nothing to put back.
|
|
2954
|
+
onBackoff: async (delayMs, deadline) => {
|
|
2955
|
+
if (!liveIsDisplaced) return delay(delayMs);
|
|
2956
|
+
await renameThroughTransientHolder(asidePath!, liveDirPath, { deadline });
|
|
2957
|
+
liveIsDisplaced = false;
|
|
2958
|
+
await syncRenameParents(asidePath!, liveDirPath);
|
|
2959
|
+
await delay(delayMs);
|
|
2960
|
+
await renameThroughTransientHolder(liveDirPath, asidePath!, { deadline });
|
|
2961
|
+
liveIsDisplaced = true;
|
|
2962
|
+
await syncRenameParents(liveDirPath, asidePath!);
|
|
2963
|
+
},
|
|
2964
|
+
});
|
|
2965
|
+
} catch (error) {
|
|
2966
|
+
// Whether the journal can still carry this activation forward, decided BEFORE compensation removes the
|
|
2967
|
+
// evidence it is read from. Only a first-ever deploy qualifies: `restoreLive` leaves the live path
|
|
2968
|
+
// absent, and recovery reads absent-plus-complete-candidate as a roll forward. A component that
|
|
2969
|
+
// already had a tree gets that tree back and loses its rollback record with it, so the next settle
|
|
2970
|
+
// reads live-plus-candidate-with-no-record and returns the artifact to dormant whatever the journal
|
|
2971
|
+
// says — keeping it there defers the same verdict to the next start and strands config until then.
|
|
2972
|
+
const recoveryCanRollForward = priorAbsentRecordPath !== undefined;
|
|
2973
|
+
await compensate(error, pendingEffect, restoreLive, application);
|
|
2974
|
+
let configRestored = true;
|
|
2975
|
+
if (undoAfterJournal) {
|
|
2976
|
+
configRestored = await undoAfterJournal().then(
|
|
2977
|
+
() => true,
|
|
2978
|
+
(undoError) => {
|
|
2979
|
+
application.logger.warn(
|
|
2980
|
+
`Restored ${application.name} after a failed activation but could not restore its root config, ` +
|
|
2981
|
+
`which still names deployment ${deploymentId}` +
|
|
2982
|
+
(recoveryCanRollForward
|
|
2983
|
+
? '; keeping its activation journal so recovery can roll the certified build forward instead:'
|
|
2984
|
+
: '. The certified build stays dormant and the previous release stays live, so the two ' +
|
|
2985
|
+
'disagree until an operator republishes the component or activates it again:'),
|
|
2986
|
+
undoError
|
|
2987
|
+
);
|
|
2988
|
+
return false;
|
|
2989
|
+
}
|
|
2990
|
+
);
|
|
2991
|
+
}
|
|
2992
|
+
// Kept only where it changes the outcome. Config is stranded either way for a component that already
|
|
2993
|
+
// had a tree — the durable-config window #2315 step 3 closes — and a journal that recovery will only
|
|
2994
|
+
// settle back to dormant buys nothing for holding it.
|
|
2995
|
+
if (configRestored || !recoveryCanRollForward) await returnToDormant();
|
|
2996
|
+
throw error;
|
|
2997
|
+
}
|
|
2998
|
+
|
|
2999
|
+
// Past the point of no return: each failure below leaves a state recovery settles forward, so they are
|
|
3000
|
+
// logged, not thrown.
|
|
3001
|
+
let swapDurable = true;
|
|
3002
|
+
try {
|
|
3003
|
+
await syncRenameParents(candidateDirPath, liveDirPath);
|
|
3004
|
+
} catch (error) {
|
|
3005
|
+
// The rename may not have reached storage. Retiring the record and removing the journal WOULD reach
|
|
3006
|
+
// it, and a power loss then leaves no live entry, no rollback record, and nothing saying to roll
|
|
3007
|
+
// forward. Both are skipped so the journal carries the activation to the next start.
|
|
3008
|
+
swapDurable = false;
|
|
3009
|
+
application.logger.warn(`Deployed ${application.name} but could not flush the swap to storage:`, error);
|
|
3010
|
+
}
|
|
3011
|
+
// The tree moved, so any dependency link that named its build path is now dangling.
|
|
3012
|
+
await repairRelocatedDependencyLinks(liveDirPath, candidateDirPath);
|
|
3013
|
+
const settledRecord = asidePath ?? priorAbsentRecordPath!;
|
|
3014
|
+
let retired = false;
|
|
3015
|
+
// Skipped entirely when the swap is not known to be on storage, so the journal below survives.
|
|
3016
|
+
if (swapDurable) {
|
|
3017
|
+
try {
|
|
3018
|
+
const retiredMarkerPath = await retireExtractionAside(settledRecord);
|
|
3019
|
+
// Retiring only MARKS the displaced tree disposable. Without this sweep the tree every deploy
|
|
3020
|
+
// displaces stays under `.deploy-aside/<component>` forever, so the components root grows by a
|
|
3021
|
+
// whole component version per deploy.
|
|
3022
|
+
await cleanupExtractionPaths(application, asideStagingDir, new Set([settledRecord, retiredMarkerPath]));
|
|
3023
|
+
// Before the journal goes: if the journal's removal persists but the record's does not, startup sees
|
|
3024
|
+
// an in-progress aside with no journal and the legacy pass restores the old tree over the new one.
|
|
3025
|
+
await syncDirectory(asideStagingDir);
|
|
3026
|
+
await syncDirectory(dirname(asideStagingDir));
|
|
3027
|
+
retired = true;
|
|
3028
|
+
} catch (error) {
|
|
3029
|
+
application.logger.warn(`Deployed ${application.name} but could not retire its rollback record:`, error);
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
// The journal goes LAST, and only once the rollback record is settled: removing it while an
|
|
3033
|
+
// `.in-progress-*` record still names the displaced tree lets the legacy pass restore the old tree.
|
|
3034
|
+
if (retired) {
|
|
3035
|
+
await rm(journalPath, { force: true }).catch((error) =>
|
|
3036
|
+
application.logger.warn(`Deployed ${application.name} but could not remove its activation journal:`, error)
|
|
3037
|
+
);
|
|
3038
|
+
await rm(deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((error) =>
|
|
3039
|
+
application.logger.warn(`Deployed ${application.name} but could not clean up its staging directory:`, error)
|
|
3040
|
+
);
|
|
3041
|
+
await rmdir(dirname(deploymentDirPath)).catch(() => {});
|
|
3042
|
+
}
|
|
3043
|
+
}
|
|
3044
|
+
|
|
3045
|
+
/**
|
|
3046
|
+
* Marks a failure where compensation ITSELF failed, so the previous version is not back and the live path
|
|
3047
|
+
* may be absent. The candidate, its `.complete` marker and its journal are then the only way back — recovery
|
|
3048
|
+
* rolls that state forward — so they must survive, and the caller keys on this to skip discarding them.
|
|
3049
|
+
*/
|
|
3050
|
+
const COMPENSATION_INCOMPLETE = Symbol('compensationIncomplete');
|
|
3051
|
+
|
|
3052
|
+
function compensationIncomplete(error: unknown): boolean {
|
|
3053
|
+
return Boolean((error as any)?.[COMPENSATION_INCOMPLETE]);
|
|
3054
|
+
}
|
|
3055
|
+
|
|
3056
|
+
/**
|
|
3057
|
+
* Undo an activation effect, folding a compensation failure into the original error rather than replacing
|
|
3058
|
+
* it — the first error is what the operator needs, the second is why the node still needs attention.
|
|
3059
|
+
*/
|
|
3060
|
+
async function compensate(
|
|
3061
|
+
error: unknown,
|
|
3062
|
+
what: string,
|
|
3063
|
+
undo: () => Promise<void>,
|
|
3064
|
+
application: Application
|
|
3065
|
+
): Promise<void> {
|
|
3066
|
+
try {
|
|
3067
|
+
await undo();
|
|
3068
|
+
} catch (undoError) {
|
|
3069
|
+
// Whatever blocked the original operation plausibly blocks its undo too — a rename into a path
|
|
3070
|
+
// something else holds open fails the same way twice.
|
|
3071
|
+
const failure = new AggregateError(
|
|
3072
|
+
[error, undoError],
|
|
3073
|
+
`Failed to ${what} for ${application.name}: ${errorMessage(error)}; ` +
|
|
3074
|
+
`also failed to restore the previous version: ${errorMessage(undoError)}`
|
|
3075
|
+
);
|
|
3076
|
+
(failure as any)[COMPENSATION_INCOMPLETE] = true;
|
|
3077
|
+
throw failure;
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
/**
|
|
3082
|
+
* Re-point dependency links that name the candidate's build path, now that it has become live. npm links a
|
|
3083
|
+
* `file:` dependency relatively on POSIX (survives the rename) but as an ABSOLUTE junction on Windows, which
|
|
3084
|
+
* then names a staging path that no longer exists. Rewriting beats `--install-links`, which would change
|
|
3085
|
+
* dependency semantics on every platform to fix one.
|
|
3086
|
+
*/
|
|
3087
|
+
async function repairRelocatedDependencyLinks(liveDirPath: string, builtAtPath: string): Promise<void> {
|
|
3088
|
+
const relinkOne = async (entryPath: string) => {
|
|
3089
|
+
let target: string;
|
|
3090
|
+
try {
|
|
3091
|
+
target = await readlink(entryPath);
|
|
3092
|
+
} catch {
|
|
3093
|
+
return;
|
|
3094
|
+
}
|
|
3095
|
+
const normalized = stripExtendedLengthPrefix(target);
|
|
3096
|
+
// Containment, not a prefix match: `startsWith` classifies `<build>-shared` as inside `<build>` and
|
|
3097
|
+
// would rewrite it to an unrelated live path.
|
|
3098
|
+
const within = relative(builtAtPath, normalized);
|
|
3099
|
+
if (within.startsWith('..') || isAbsolute(within)) return;
|
|
3100
|
+
const repaired = join(liveDirPath, within);
|
|
3101
|
+
// The replacement is created BEFORE the old link is dropped, and swapped in by rename. A
|
|
3102
|
+
// remove-then-create loses the dependency outright when the create fails.
|
|
3103
|
+
const stagedLink = `${entryPath}.relink-${process.pid}-${randomUUID()}`;
|
|
3104
|
+
try {
|
|
3105
|
+
await symlink(repaired, stagedLink, 'junction');
|
|
3106
|
+
try {
|
|
3107
|
+
await rename(stagedLink, entryPath);
|
|
3108
|
+
} catch (renameError) {
|
|
3109
|
+
// Windows cannot rename over an existing junction, so the old one has to go first — and if the
|
|
3110
|
+
// second rename then fails the same way, the original target is put back rather than leaving
|
|
3111
|
+
// nothing behind.
|
|
3112
|
+
if (process.platform !== 'win32') throw renameError;
|
|
3113
|
+
await rm(entryPath, { recursive: true, force: true });
|
|
3114
|
+
try {
|
|
3115
|
+
await rename(stagedLink, entryPath);
|
|
3116
|
+
} catch (secondError) {
|
|
3117
|
+
await symlink(normalized, entryPath, 'junction').catch(() => {});
|
|
3118
|
+
throw secondError;
|
|
3119
|
+
}
|
|
3120
|
+
}
|
|
3121
|
+
} catch (error) {
|
|
3122
|
+
await rm(stagedLink, { recursive: true, force: true }).catch(() => {});
|
|
3123
|
+
logger.warn(`Could not re-point ${entryPath} after activation: ${errorMessage(error)}`);
|
|
3124
|
+
}
|
|
3125
|
+
};
|
|
3126
|
+
|
|
3127
|
+
// ONE bounded pool over a shared queue, not per-directory concurrency: bounding each parent
|
|
3128
|
+
// independently let every one of N workers start N more, so a deep pnpm or monorepo tree fanned out to
|
|
3129
|
+
// thousands of simultaneous opens. An EMFILE there would surface as skipped subtrees and dangling links.
|
|
3130
|
+
const pending: string[] = [join(liveDirPath, 'node_modules')];
|
|
3131
|
+
let active = 0;
|
|
3132
|
+
const visit = async (dirPath: string): Promise<void> => {
|
|
3133
|
+
let entries;
|
|
3134
|
+
try {
|
|
3135
|
+
entries = await readdir(dirPath, { withFileTypes: true });
|
|
3136
|
+
} catch (error) {
|
|
3137
|
+
// Not silent. A missing directory is ordinary — the tree has no `node_modules`, or an install
|
|
3138
|
+
// removed one — but an EACCES or EMFILE here means links under it were never examined, which is
|
|
3139
|
+
// exactly the state that leaves a component running against a dangling dependency.
|
|
3140
|
+
if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') {
|
|
3141
|
+
logger.warn(`Could not scan ${dirPath} for links to re-point after activation:`, errorForLog(error));
|
|
3142
|
+
}
|
|
3143
|
+
return;
|
|
3144
|
+
}
|
|
3145
|
+
// Every directory, not just `@scope` containers and nested `node_modules`: a dependency installed from
|
|
3146
|
+
// outside the tree can be linked from deeper in.
|
|
3147
|
+
for (const entry of entries) {
|
|
3148
|
+
const entryPath = join(dirPath, entry.name);
|
|
3149
|
+
if (entry.isSymbolicLink()) await relinkOne(entryPath);
|
|
3150
|
+
else if (entry.isDirectory()) pending.push(entryPath);
|
|
3151
|
+
}
|
|
3152
|
+
};
|
|
3153
|
+
const worker = async (): Promise<void> => {
|
|
3154
|
+
for (;;) {
|
|
3155
|
+
const next = pending.pop();
|
|
3156
|
+
if (next === undefined) {
|
|
3157
|
+
// Another worker may still be about to enqueue children, so only stop once nothing is in flight.
|
|
3158
|
+
if (active === 0) return;
|
|
3159
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
3160
|
+
continue;
|
|
3161
|
+
}
|
|
3162
|
+
active++;
|
|
3163
|
+
try {
|
|
3164
|
+
await visit(next);
|
|
3165
|
+
} finally {
|
|
3166
|
+
active--;
|
|
3167
|
+
}
|
|
3168
|
+
}
|
|
3169
|
+
};
|
|
3170
|
+
await Promise.all(Array.from({ length: LINK_REPAIR_CONCURRENCY }, worker));
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
/** Windows junction targets come back with an extended-length `\\?\` prefix that plain paths never have. */
|
|
3174
|
+
function stripExtendedLengthPrefix(target: string): string {
|
|
3175
|
+
return target.startsWith('\\\\?\\') ? target.slice(4) : target;
|
|
3176
|
+
}
|
|
694
3177
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
3178
|
+
/** Remove a candidate's whole deployment directory, best-effort — it is never the last good copy. */
|
|
3179
|
+
async function discardCandidate(application: Application, deploymentId: string): Promise<void> {
|
|
3180
|
+
const deploymentDirPath = candidateDeploymentDirPath(application.dirPath, deploymentId);
|
|
3181
|
+
await rm(deploymentDirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((error) =>
|
|
3182
|
+
application.logger.warn(`Failed to remove the abandoned deploy candidate at ${deploymentDirPath}:`, error)
|
|
3183
|
+
);
|
|
3184
|
+
// And the staging root itself once nothing is in it, so an idle install leaves the components root as
|
|
3185
|
+
// it found it. ENOTEMPTY just means a concurrent deploy still owns a candidate.
|
|
3186
|
+
await rmdir(dirname(deploymentDirPath)).catch(() => {});
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
/**
|
|
3190
|
+
* Build a deploy candidate at `.deploy-staging/<deploymentId>/<component>`, leaving the live tree
|
|
3191
|
+
* completely untouched — this is what lets the previous version keep serving through the clone, the
|
|
3192
|
+
* extraction and the dependency install.
|
|
3193
|
+
*
|
|
3194
|
+
* Failure needs no compensation, which is the whole point: nothing about the live component was modified,
|
|
3195
|
+
* so the abandoned candidate is simply removed and the error propagates.
|
|
3196
|
+
*/
|
|
3197
|
+
export async function buildCandidateApplication(
|
|
3198
|
+
application: Application,
|
|
3199
|
+
deploymentId: string,
|
|
3200
|
+
options: { rejectLinkSource?: boolean } = {}
|
|
3201
|
+
): Promise<string> {
|
|
3202
|
+
const deploymentDirPath = candidateDeploymentDirPath(application.dirPath, deploymentId);
|
|
3203
|
+
const candidateDirPath = candidateApplicationPath(application.dirPath, deploymentId);
|
|
3204
|
+
await ensureSecureStagingDirectory(dirname(deploymentDirPath));
|
|
3205
|
+
await claimDeploymentDirectory(deploymentDirPath, application.name);
|
|
3206
|
+
try {
|
|
3207
|
+
// Replaced, not extracted into: a prior attempt on this id may have left a partial tree.
|
|
3208
|
+
await rm(candidateDirPath, { recursive: true, force: true });
|
|
3209
|
+
const resolved = await resolveApplicationTarball(application);
|
|
3210
|
+
if (resolved.kind === 'link' && options.rejectLinkSource) {
|
|
3211
|
+
// What the operator asked for, not a server fault: the same component deploys immediately.
|
|
3212
|
+
throw new ClientError(
|
|
3213
|
+
`Cannot stage ${application.name} from ${application.packageIdentifier}: a 'file:' directory is linked ` +
|
|
3214
|
+
`rather than copied, so the bytes activated later are not the bytes this build certified`
|
|
3215
|
+
);
|
|
3216
|
+
}
|
|
3217
|
+
if (resolved.kind === 'link') {
|
|
3218
|
+
// A `file:` directory becomes a symlink AT THE CANDIDATE PATH, so it is validated and swapped in
|
|
3219
|
+
// like any other candidate instead of appearing at the live path unvalidated.
|
|
3220
|
+
await symlink(resolved.sourceDirPath, candidateDirPath, 'dir');
|
|
3221
|
+
} else {
|
|
3222
|
+
const { tarball, tarballPath, shouldDeleteTarball } = resolved;
|
|
3223
|
+
try {
|
|
3224
|
+
await extractTarballInto(tarball, candidateDirPath, deploymentDirPath);
|
|
3225
|
+
} finally {
|
|
3226
|
+
if (!tarball.destroyed) tarball.destroy();
|
|
3227
|
+
if (shouldDeleteTarball && tarballPath) {
|
|
3228
|
+
await rm(tarballPath, { force: true }).catch((error) =>
|
|
3229
|
+
application.logger.warn(`Failed to remove temporary package ${tarballPath}:`, error)
|
|
706
3230
|
);
|
|
707
3231
|
}
|
|
708
|
-
tarballPath = await runNpmPack(application, packArgs, parentDirPath, application.gitCredentialEnv);
|
|
709
3232
|
}
|
|
710
|
-
shouldDeleteTarball = true;
|
|
711
|
-
tarball = createReadStream(tarballPath);
|
|
712
3233
|
}
|
|
3234
|
+
// The credential socket only has to be up for extraction — that is where npm resolves and clones a
|
|
3235
|
+
// git-reference package. Closed BEFORE the install so the dependency tree's own install scripts,
|
|
3236
|
+
// which are arbitrary code from the registry running as this uid, cannot ask the helper for the
|
|
3237
|
+
// deployer's git token. `prepareApplication`'s finally still calls this; it is idempotent.
|
|
3238
|
+
await application.cleanupGitCredentialSession();
|
|
3239
|
+
await installApplication(application, candidateDirPath);
|
|
3240
|
+
return candidateDirPath;
|
|
3241
|
+
} catch (error) {
|
|
3242
|
+
await discardCandidate(application, deploymentId);
|
|
3243
|
+
throw error;
|
|
713
3244
|
}
|
|
3245
|
+
}
|
|
714
3246
|
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
//
|
|
741
|
-
|
|
742
|
-
|
|
3247
|
+
async function recoverOrCleanupStaleExtractionPaths(
|
|
3248
|
+
application: ExtractionContext,
|
|
3249
|
+
asideStagingDir: string
|
|
3250
|
+
): Promise<void> {
|
|
3251
|
+
const entries = await readdir(asideStagingDir, { withFileTypes: true });
|
|
3252
|
+
const entryNames = new Set(entries.map((entry) => entry.name));
|
|
3253
|
+
const paths = new Set<string>(entries.map((entry) => join(asideStagingDir, entry.name)));
|
|
3254
|
+
const recoveryRecords = entries
|
|
3255
|
+
.filter(
|
|
3256
|
+
(entry) =>
|
|
3257
|
+
isExtractionRecoveryRecord(entry) &&
|
|
3258
|
+
entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX) &&
|
|
3259
|
+
!entryNames.has(`${RETIRED_ASIDE_PREFIX}${entry.name.slice(IN_PROGRESS_ASIDE_PREFIX.length)}`)
|
|
3260
|
+
)
|
|
3261
|
+
.map((entry) => ({
|
|
3262
|
+
entry,
|
|
3263
|
+
priorStateAbsent: isPriorAbsentRecoveryRecord(entry),
|
|
3264
|
+
timestamp: extractionAsideTimestamp(entry.name),
|
|
3265
|
+
}))
|
|
3266
|
+
.filter(({ timestamp }) => Number.isFinite(timestamp))
|
|
3267
|
+
.sort((left, right) => right.timestamp - left.timestamp);
|
|
3268
|
+
const recoveryRecord =
|
|
3269
|
+
recoveryRecords.find(({ priorStateAbsent }) => !priorStateAbsent) ??
|
|
3270
|
+
recoveryRecords.find(({ priorStateAbsent }) => priorStateAbsent);
|
|
3271
|
+
if (recoveryRecord) {
|
|
3272
|
+
// A journal outranks the record. Without this the pass restores the tree a completed activation
|
|
3273
|
+
// displaced, back over the candidate it committed — the inversion the journal exists to prevent, and
|
|
3274
|
+
// reachable on any thread whose settlement did not run or did not succeed. Enforced HERE, at the one
|
|
3275
|
+
// place a tree is restored, so every entry point is covered by construction rather than by each
|
|
3276
|
+
// caller remembering to settle first — and so a component with nothing left to restore still loads.
|
|
3277
|
+
const journaled = await journaledDeploymentForComponent(dirname(application.dirPath), application.name);
|
|
3278
|
+
if (journaled) {
|
|
3279
|
+
throw new Error(
|
|
3280
|
+
`Refusing to restore ${application.name} from ${recoveryRecord.entry.name}: the interrupted ` +
|
|
3281
|
+
`activation in ${journaled} is not settled, and its journal is the only record of which tree ` +
|
|
3282
|
+
`is current. If that journal names no component at all it cannot settle itself; remove that ` +
|
|
3283
|
+
`directory once you have determined which tree is current.`
|
|
3284
|
+
);
|
|
743
3285
|
}
|
|
3286
|
+
const recoveryPath = join(asideStagingDir, recoveryRecord.entry.name);
|
|
3287
|
+
// Retire the losing candidates durably; a cleanup that fails must not let a later
|
|
3288
|
+
// pass adopt one of them and restore an older tree over the one recovered here.
|
|
3289
|
+
for (const { entry } of recoveryRecords) {
|
|
3290
|
+
if (entry === recoveryRecord.entry) continue;
|
|
3291
|
+
paths.add(await retireExtractionAside(join(asideStagingDir, entry.name)));
|
|
3292
|
+
}
|
|
3293
|
+
await rollbackExtractedDirectory(
|
|
3294
|
+
application,
|
|
3295
|
+
asideStagingDir,
|
|
3296
|
+
recoveryRecord.priorStateAbsent ? undefined : recoveryPath,
|
|
3297
|
+
paths,
|
|
3298
|
+
false
|
|
3299
|
+
);
|
|
3300
|
+
application.logger.warn(
|
|
3301
|
+
(recoveryRecord.priorStateAbsent
|
|
3302
|
+
? `Removed the partial ${application.name} component directory after an interrupted first deploy`
|
|
3303
|
+
: `Recovered the previous ${application.name} component directory after an interrupted deploy`) +
|
|
3304
|
+
(recoveryRecords.length > 1 ? `; discarded ${recoveryRecords.length - 1} older recovery candidates` : '')
|
|
3305
|
+
);
|
|
3306
|
+
return;
|
|
744
3307
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
3308
|
+
await cleanupExtractionPaths(application, asideStagingDir, paths);
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
function isPriorAbsentRecoveryRecord(entry: { isFile(): boolean; name: string }): boolean {
|
|
3312
|
+
return entry.isFile() && entry.name.endsWith(PRIOR_ABSENT_RECORD_SUFFIX);
|
|
3313
|
+
}
|
|
3314
|
+
|
|
3315
|
+
function isExtractionRecoveryRecord(entry: {
|
|
3316
|
+
isDirectory(): boolean;
|
|
3317
|
+
isFile(): boolean;
|
|
3318
|
+
isSymbolicLink(): boolean;
|
|
3319
|
+
name: string;
|
|
3320
|
+
}): boolean {
|
|
3321
|
+
return entry.isDirectory() || entry.isSymbolicLink() || isPriorAbsentRecoveryRecord(entry);
|
|
3322
|
+
}
|
|
3323
|
+
|
|
3324
|
+
function extractionAsideTimestamp(name: string): number {
|
|
3325
|
+
const timestampEnd = name.indexOf('-', IN_PROGRESS_ASIDE_PREFIX.length);
|
|
3326
|
+
if (timestampEnd < 0) return Number.NaN;
|
|
3327
|
+
return Number(name.slice(IN_PROGRESS_ASIDE_PREFIX.length, timestampEnd));
|
|
3328
|
+
}
|
|
3329
|
+
|
|
3330
|
+
export async function recoverInterruptedComponentExtractions(
|
|
3331
|
+
componentsRootDirPath: string
|
|
3332
|
+
): Promise<Map<string, Error>> {
|
|
3333
|
+
const stagingRoot = join(componentsRootDirPath, ASIDE_STAGING_DIR);
|
|
3334
|
+
let entries;
|
|
748
3335
|
try {
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
const extracted = await readdir(application.dirPath, { withFileTypes: true });
|
|
753
|
-
if (extracted.length === 1 && extracted[0].isDirectory()) {
|
|
754
|
-
const topLevelDirPath = join(application.dirPath, extracted[0].name);
|
|
755
|
-
await mkdir(asideStagingDir, { recursive: true });
|
|
756
|
-
const tempDirPath = await mkdtemp(join(asideStagingDir, '.normalize-'));
|
|
757
|
-
await cp(topLevelDirPath, tempDirPath, { recursive: true });
|
|
758
|
-
await rm(topLevelDirPath, { recursive: true, force: true });
|
|
759
|
-
await cp(tempDirPath, application.dirPath, { recursive: true });
|
|
760
|
-
await rm(tempDirPath, { recursive: true, force: true });
|
|
3336
|
+
const stagingStat = await lstat(stagingRoot);
|
|
3337
|
+
if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) {
|
|
3338
|
+
throw new Error(`Component deploy staging path is not a directory: ${stagingRoot}`);
|
|
761
3339
|
}
|
|
3340
|
+
entries = await readdir(stagingRoot, { withFileTypes: true });
|
|
762
3341
|
} catch (error) {
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
3342
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return new Map();
|
|
3343
|
+
throw error;
|
|
3344
|
+
}
|
|
3345
|
+
const failedComponents = new Map<string, Error>();
|
|
3346
|
+
await Promise.all(
|
|
3347
|
+
entries
|
|
3348
|
+
.filter((entry) => entry.isDirectory())
|
|
3349
|
+
.map(async (entry) => {
|
|
3350
|
+
try {
|
|
3351
|
+
await recoverInterruptedComponentExtraction(componentsRootDirPath, entry.name, false);
|
|
3352
|
+
} catch (error) {
|
|
3353
|
+
const recoveryError = error instanceof Error ? error : new Error(String(error));
|
|
3354
|
+
failedComponents.set(entry.name, recoveryError);
|
|
3355
|
+
const deferred = recoveryError instanceof ComponentPreparationLockTimeoutError;
|
|
3356
|
+
logger[deferred ? 'warn' : 'error'](
|
|
3357
|
+
`${deferred ? 'Deferring' : 'Not loading'} ${entry.name} because its interrupted component deployment ` +
|
|
3358
|
+
`${deferred ? 'is still being prepared' : 'could not be recovered'}:`,
|
|
3359
|
+
errorForLog(recoveryError)
|
|
3360
|
+
);
|
|
3361
|
+
}
|
|
3362
|
+
})
|
|
3363
|
+
);
|
|
3364
|
+
return failedComponents;
|
|
3365
|
+
}
|
|
3366
|
+
|
|
3367
|
+
export async function recoverInterruptedComponentExtraction(
|
|
3368
|
+
componentsRootDirPath: string,
|
|
3369
|
+
componentName: string,
|
|
3370
|
+
waitForPreparation = true,
|
|
3371
|
+
waitTimeoutMs = COMPONENT_RECOVERY_WAIT_TIMEOUT_MS
|
|
3372
|
+
): Promise<void> {
|
|
3373
|
+
const componentDirPath = join(componentsRootDirPath, componentName);
|
|
3374
|
+
await withComponentPreparationLock(
|
|
3375
|
+
componentDirPath,
|
|
3376
|
+
async () => {
|
|
3377
|
+
const asideStagingDir = extractionStagingDirectory(componentDirPath);
|
|
3378
|
+
try {
|
|
3379
|
+
await lstat(asideStagingDir);
|
|
3380
|
+
} catch (error) {
|
|
3381
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
|
|
3382
|
+
throw error;
|
|
3383
|
+
}
|
|
3384
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
3385
|
+
await recoverOrCleanupStaleExtractionPaths(
|
|
3386
|
+
{ name: componentName, dirPath: componentDirPath, logger },
|
|
3387
|
+
asideStagingDir
|
|
769
3388
|
);
|
|
3389
|
+
},
|
|
3390
|
+
{
|
|
3391
|
+
timeoutMs: waitForPreparation ? waitTimeoutMs : COMPONENT_RECOVERY_TRY_TIMEOUT_MS,
|
|
3392
|
+
purpose: COMPONENT_RECOVERY_LOCK_PURPOSE,
|
|
3393
|
+
renewTimeoutWhileOwnerAlive: waitForPreparation,
|
|
3394
|
+
onWait: (owner) => {
|
|
3395
|
+
logger.info(
|
|
3396
|
+
`Waiting to settle component deployment state for ${componentName}` +
|
|
3397
|
+
(owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '')
|
|
3398
|
+
);
|
|
3399
|
+
},
|
|
3400
|
+
isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId),
|
|
770
3401
|
}
|
|
3402
|
+
);
|
|
3403
|
+
}
|
|
3404
|
+
|
|
3405
|
+
export async function retireComponentExtractionStaging(
|
|
3406
|
+
componentDirPath: string,
|
|
3407
|
+
componentName = basename(componentDirPath),
|
|
3408
|
+
componentLogger: Logger = logger
|
|
3409
|
+
): Promise<void> {
|
|
3410
|
+
const asideStagingDir = extractionStagingDirectory(componentDirPath);
|
|
3411
|
+
let entries;
|
|
3412
|
+
try {
|
|
3413
|
+
await lstat(asideStagingDir);
|
|
3414
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
3415
|
+
entries = await readdir(asideStagingDir, { withFileTypes: true });
|
|
3416
|
+
} catch (error) {
|
|
3417
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
|
|
771
3418
|
throw error;
|
|
772
3419
|
}
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
3420
|
+
const paths = new Set<string>(entries.map((entry) => join(asideStagingDir, entry.name)));
|
|
3421
|
+
for (const entry of entries) {
|
|
3422
|
+
if (!isExtractionRecoveryRecord(entry) || !entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX)) continue;
|
|
3423
|
+
const markerPath = retiredMarkerForAside(join(asideStagingDir, entry.name));
|
|
3424
|
+
try {
|
|
3425
|
+
await writeFile(markerPath, '', { flag: 'wx', mode: 0o600 });
|
|
3426
|
+
} catch (error) {
|
|
3427
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
3428
|
+
}
|
|
3429
|
+
paths.add(markerPath);
|
|
778
3430
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
// its worker is still live is swept by the next deploy. The failure is expected in
|
|
785
|
-
// the live-worker case, so it's logged at trace rather than as a warning.
|
|
786
|
-
let settled = false;
|
|
787
|
-
const transaction: ExtractionTransaction = {
|
|
788
|
-
async commit() {
|
|
789
|
-
if (settled) return;
|
|
790
|
-
settled = true;
|
|
791
|
-
await cleanupExtractionStaging(application, asideStagingDir);
|
|
792
|
-
},
|
|
793
|
-
async rollback() {
|
|
794
|
-
if (settled) return;
|
|
795
|
-
settled = true;
|
|
796
|
-
await rollbackExtractedDirectory(application, asideStagingDir, asidePath);
|
|
797
|
-
},
|
|
798
|
-
};
|
|
799
|
-
if (deferCommit) return transaction;
|
|
800
|
-
await transaction.commit();
|
|
3431
|
+
await cleanupExtractionPaths(
|
|
3432
|
+
{ name: componentName, dirPath: componentDirPath, logger: componentLogger },
|
|
3433
|
+
asideStagingDir,
|
|
3434
|
+
paths
|
|
3435
|
+
);
|
|
801
3436
|
}
|
|
802
3437
|
|
|
803
|
-
async function
|
|
3438
|
+
export async function dropComponentDirectory(
|
|
3439
|
+
componentDirPath: string,
|
|
3440
|
+
componentName = basename(componentDirPath),
|
|
3441
|
+
componentLogger: Logger = logger
|
|
3442
|
+
): Promise<void> {
|
|
3443
|
+
await retireComponentExtractionStaging(componentDirPath, componentName, componentLogger);
|
|
3444
|
+
const asideStagingDir = extractionStagingDirectory(componentDirPath);
|
|
3445
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
3446
|
+
const droppedPath = join(asideStagingDir, `.dropped-${process.pid}-${Date.now()}-${randomUUID()}`);
|
|
3447
|
+
try {
|
|
3448
|
+
await rename(componentDirPath, droppedPath);
|
|
3449
|
+
} catch (error) {
|
|
3450
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
3451
|
+
}
|
|
3452
|
+
await cleanupExtractionPaths(
|
|
3453
|
+
{ name: componentName, dirPath: componentDirPath, logger: componentLogger },
|
|
3454
|
+
asideStagingDir,
|
|
3455
|
+
new Set([droppedPath])
|
|
3456
|
+
);
|
|
3457
|
+
// A dropped component has no next deploy to bound its dormant builds.
|
|
804
3458
|
try {
|
|
805
|
-
await
|
|
3459
|
+
await pruneDormantBuilds(componentName, await dormantBuildsOf(dirname(componentDirPath), componentName), 0);
|
|
806
3460
|
} catch (error) {
|
|
807
|
-
|
|
3461
|
+
componentLogger.warn(
|
|
3462
|
+
`Dropped ${componentName} but could not reclaim its dormant staged builds:`,
|
|
3463
|
+
errorForLog(error)
|
|
3464
|
+
);
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
async function cleanupExtractionPaths(
|
|
3469
|
+
application: ExtractionContext,
|
|
3470
|
+
asideStagingDir: string,
|
|
3471
|
+
paths: Set<string>
|
|
3472
|
+
): Promise<void> {
|
|
3473
|
+
const retiredMarkers: string[] = [];
|
|
3474
|
+
for (const path of paths) {
|
|
3475
|
+
if (basename(path).startsWith(RETIRED_ASIDE_PREFIX)) {
|
|
3476
|
+
retiredMarkers.push(path);
|
|
3477
|
+
continue;
|
|
3478
|
+
}
|
|
3479
|
+
try {
|
|
3480
|
+
await rm(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
3481
|
+
} catch (error) {
|
|
3482
|
+
application.logger.trace?.(
|
|
3483
|
+
`Cleanup of previous ${application.name} component directory deferred: ${errorMessage(error)}`
|
|
3484
|
+
);
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
for (const markerPath of retiredMarkers) {
|
|
3488
|
+
const asidePath = join(
|
|
3489
|
+
asideStagingDir,
|
|
3490
|
+
`${IN_PROGRESS_ASIDE_PREFIX}${basename(markerPath).slice(RETIRED_ASIDE_PREFIX.length)}`
|
|
3491
|
+
);
|
|
3492
|
+
try {
|
|
3493
|
+
await access(asidePath, constants.F_OK);
|
|
3494
|
+
continue;
|
|
3495
|
+
} catch (error) {
|
|
3496
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') continue;
|
|
3497
|
+
}
|
|
3498
|
+
await rm(markerPath, { force: true }).catch((error) =>
|
|
3499
|
+
application.logger.trace?.(
|
|
3500
|
+
`Cleanup of previous ${application.name} component directory deferred: ${errorMessage(error)}`
|
|
3501
|
+
)
|
|
3502
|
+
);
|
|
808
3503
|
}
|
|
3504
|
+
await rmdir(asideStagingDir).catch((error) => {
|
|
3505
|
+
if (!['ENOENT', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) {
|
|
3506
|
+
application.logger.trace?.(
|
|
3507
|
+
`Cleanup of ${application.name} deploy staging directory deferred: ${errorMessage(error)}`
|
|
3508
|
+
);
|
|
3509
|
+
}
|
|
3510
|
+
});
|
|
809
3511
|
}
|
|
810
3512
|
|
|
811
3513
|
async function rollbackExtractedDirectory(
|
|
812
|
-
application:
|
|
3514
|
+
application: ExtractionContext,
|
|
813
3515
|
asideStagingDir: string,
|
|
814
|
-
asidePath: string | undefined
|
|
3516
|
+
asidePath: string | undefined,
|
|
3517
|
+
transactionPaths: Set<string>,
|
|
3518
|
+
retainReplacement: boolean
|
|
815
3519
|
): Promise<void> {
|
|
816
|
-
await
|
|
817
|
-
const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES', 'EBUSY']);
|
|
818
|
-
const
|
|
3520
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
3521
|
+
const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'ENOTDIR', 'EISDIR', 'EPERM', 'EACCES', 'EBUSY']);
|
|
3522
|
+
const displaceCurrentDirectorySync = (): string | undefined => {
|
|
819
3523
|
const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`);
|
|
3524
|
+
try {
|
|
3525
|
+
renameSync(application.dirPath, displacedPath);
|
|
3526
|
+
transactionPaths.add(displacedPath);
|
|
3527
|
+
return displacedPath;
|
|
3528
|
+
} catch (error) {
|
|
3529
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
|
3530
|
+
throw error;
|
|
3531
|
+
}
|
|
3532
|
+
};
|
|
3533
|
+
const displaceCurrentDirectory = async (): Promise<string | undefined> => {
|
|
3534
|
+
const retryDeadline = Date.now() + 5000;
|
|
820
3535
|
let lastError: unknown;
|
|
821
|
-
|
|
3536
|
+
do {
|
|
3537
|
+
const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`);
|
|
822
3538
|
try {
|
|
823
3539
|
await rename(application.dirPath, displacedPath);
|
|
824
|
-
|
|
3540
|
+
transactionPaths.add(displacedPath);
|
|
3541
|
+
return displacedPath;
|
|
825
3542
|
} catch (error) {
|
|
826
3543
|
const code = (error as NodeJS.ErrnoException).code;
|
|
827
|
-
if (code === 'ENOENT') return;
|
|
3544
|
+
if (code === 'ENOENT') return undefined;
|
|
828
3545
|
if (!retryableRenameCodes.has(code ?? '')) throw error;
|
|
829
3546
|
lastError = error;
|
|
830
3547
|
await delay(10);
|
|
831
3548
|
}
|
|
832
|
-
}
|
|
3549
|
+
} while (Date.now() < retryDeadline);
|
|
833
3550
|
throw lastError;
|
|
834
3551
|
};
|
|
835
3552
|
|
|
836
|
-
await displaceCurrentDirectory();
|
|
837
3553
|
if (asidePath) {
|
|
838
|
-
|
|
3554
|
+
const asideIsSymbolicLink = (await lstat(asidePath)).isSymbolicLink();
|
|
839
3555
|
let restoreError: unknown;
|
|
840
|
-
|
|
3556
|
+
let restoreRetryDeadline: number | undefined;
|
|
3557
|
+
let fallbackDisplacedPath: string | undefined;
|
|
3558
|
+
let placeholderIdentity = await identifyRollbackPlaceholder(application.dirPath);
|
|
3559
|
+
const failRestore = async (error: unknown): Promise<never> => {
|
|
3560
|
+
try {
|
|
3561
|
+
await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity);
|
|
3562
|
+
if (retainReplacement && fallbackDisplacedPath) {
|
|
3563
|
+
const fallbackRetryDeadline = Date.now() + 5000;
|
|
3564
|
+
let fallbackRestoreError: unknown;
|
|
3565
|
+
do {
|
|
3566
|
+
let writerDisplacedPath: string | undefined;
|
|
3567
|
+
try {
|
|
3568
|
+
writerDisplacedPath = displaceCurrentDirectorySync();
|
|
3569
|
+
placeholderIdentity = undefined;
|
|
3570
|
+
renameSync(fallbackDisplacedPath, application.dirPath);
|
|
3571
|
+
fallbackRestoreError = undefined;
|
|
3572
|
+
} catch (restoreFallbackError) {
|
|
3573
|
+
fallbackRestoreError = restoreFallbackError;
|
|
3574
|
+
}
|
|
3575
|
+
if (writerDisplacedPath) {
|
|
3576
|
+
await rm(writerDisplacedPath, {
|
|
3577
|
+
recursive: true,
|
|
3578
|
+
force: true,
|
|
3579
|
+
maxRetries: 3,
|
|
3580
|
+
retryDelay: 100,
|
|
3581
|
+
});
|
|
3582
|
+
transactionPaths.delete(writerDisplacedPath);
|
|
3583
|
+
}
|
|
3584
|
+
if (
|
|
3585
|
+
fallbackRestoreError &&
|
|
3586
|
+
!retryableRenameCodes.has((fallbackRestoreError as NodeJS.ErrnoException).code ?? '')
|
|
3587
|
+
) {
|
|
3588
|
+
throw fallbackRestoreError;
|
|
3589
|
+
}
|
|
3590
|
+
if (fallbackRestoreError) {
|
|
3591
|
+
await delay(10);
|
|
3592
|
+
} else {
|
|
3593
|
+
break;
|
|
3594
|
+
}
|
|
3595
|
+
} while (Date.now() < fallbackRetryDeadline);
|
|
3596
|
+
if (fallbackRestoreError) throw fallbackRestoreError;
|
|
3597
|
+
transactionPaths.delete(fallbackDisplacedPath);
|
|
3598
|
+
transactionPaths.add(await retireExtractionAside(asidePath));
|
|
3599
|
+
}
|
|
3600
|
+
if (placeholderIdentity) {
|
|
3601
|
+
try {
|
|
3602
|
+
const current = await lstat(application.dirPath, { bigint: true });
|
|
3603
|
+
if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) {
|
|
3604
|
+
await rm(application.dirPath, {
|
|
3605
|
+
recursive: true,
|
|
3606
|
+
force: true,
|
|
3607
|
+
maxRetries: 3,
|
|
3608
|
+
retryDelay: 100,
|
|
3609
|
+
});
|
|
3610
|
+
placeholderIdentity = undefined;
|
|
3611
|
+
}
|
|
3612
|
+
} catch (placeholderError) {
|
|
3613
|
+
if ((placeholderError as NodeJS.ErrnoException).code !== 'ENOENT') throw placeholderError;
|
|
3614
|
+
}
|
|
3615
|
+
}
|
|
3616
|
+
const disposablePaths = new Set(transactionPaths);
|
|
3617
|
+
disposablePaths.delete(asidePath);
|
|
3618
|
+
await cleanupExtractionPaths(application, asideStagingDir, disposablePaths);
|
|
3619
|
+
} catch (fallbackError) {
|
|
3620
|
+
throw new AggregateError(
|
|
3621
|
+
[error, fallbackError],
|
|
3622
|
+
`Failed to restore either the previous or replacement ${application.name} component directory`
|
|
3623
|
+
);
|
|
3624
|
+
}
|
|
3625
|
+
throw new Error(
|
|
3626
|
+
`Failed to restore ${asidePath} to the live component directory ${application.dirPath}: ${errorMessage(error)}`,
|
|
3627
|
+
{ cause: error }
|
|
3628
|
+
);
|
|
3629
|
+
};
|
|
3630
|
+
do {
|
|
841
3631
|
try {
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
3632
|
+
if (placeholderIdentity) {
|
|
3633
|
+
const current = lstatSync(application.dirPath, { bigint: true });
|
|
3634
|
+
if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) {
|
|
3635
|
+
chmodSync(application.dirPath, 0o700);
|
|
3636
|
+
renameSync(asidePath, application.dirPath);
|
|
3637
|
+
} else {
|
|
3638
|
+
placeholderIdentity = undefined;
|
|
3639
|
+
await rename(asidePath, application.dirPath);
|
|
3640
|
+
}
|
|
3641
|
+
} else {
|
|
3642
|
+
await rename(asidePath, application.dirPath);
|
|
3643
|
+
}
|
|
3644
|
+
transactionPaths.delete(asidePath);
|
|
3645
|
+
await cleanupExtractionPaths(application, asideStagingDir, transactionPaths);
|
|
3646
|
+
return;
|
|
845
3647
|
} catch (error) {
|
|
846
3648
|
restoreError = error;
|
|
847
|
-
if (!retryableRenameCodes.has((error as NodeJS.ErrnoException).code ?? ''))
|
|
848
|
-
|
|
3649
|
+
if (!retryableRenameCodes.has((error as NodeJS.ErrnoException).code ?? '')) {
|
|
3650
|
+
return failRestore(error);
|
|
3651
|
+
}
|
|
3652
|
+
let displacedPath: string | undefined;
|
|
3653
|
+
const displacedPlaceholderIdentity = placeholderIdentity;
|
|
3654
|
+
try {
|
|
3655
|
+
await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity);
|
|
3656
|
+
displacedPath = await displaceCurrentDirectory();
|
|
3657
|
+
placeholderIdentity = undefined;
|
|
3658
|
+
if (displacedPath && displacedPlaceholderIdentity) {
|
|
3659
|
+
try {
|
|
3660
|
+
const displaced = await lstat(displacedPath, { bigint: true });
|
|
3661
|
+
if (
|
|
3662
|
+
displaced.dev === displacedPlaceholderIdentity.dev &&
|
|
3663
|
+
displaced.ino === displacedPlaceholderIdentity.ino
|
|
3664
|
+
) {
|
|
3665
|
+
const displacedPlaceholderPath = displacedPath;
|
|
3666
|
+
displacedPath = undefined;
|
|
3667
|
+
await rm(displacedPlaceholderPath, {
|
|
3668
|
+
recursive: true,
|
|
3669
|
+
force: true,
|
|
3670
|
+
maxRetries: 3,
|
|
3671
|
+
retryDelay: 100,
|
|
3672
|
+
});
|
|
3673
|
+
transactionPaths.delete(displacedPlaceholderPath);
|
|
3674
|
+
}
|
|
3675
|
+
} catch (placeholderCleanupError) {
|
|
3676
|
+
application.logger.trace?.(
|
|
3677
|
+
`Cleanup of the ${application.name} rollback placeholder deferred: ${errorMessage(placeholderCleanupError)}`
|
|
3678
|
+
);
|
|
3679
|
+
}
|
|
3680
|
+
}
|
|
3681
|
+
} catch (displaceError) {
|
|
3682
|
+
return failRestore(
|
|
3683
|
+
new AggregateError(
|
|
3684
|
+
[error, displaceError],
|
|
3685
|
+
`Failed to clear the live ${application.name} component directory for rollback`
|
|
3686
|
+
)
|
|
3687
|
+
);
|
|
3688
|
+
}
|
|
3689
|
+
if (displacedPath) {
|
|
3690
|
+
if (fallbackDisplacedPath) {
|
|
3691
|
+
try {
|
|
3692
|
+
await rm(displacedPath, {
|
|
3693
|
+
recursive: true,
|
|
3694
|
+
force: true,
|
|
3695
|
+
maxRetries: 3,
|
|
3696
|
+
retryDelay: 100,
|
|
3697
|
+
});
|
|
3698
|
+
transactionPaths.delete(displacedPath);
|
|
3699
|
+
} catch (cleanupError) {
|
|
3700
|
+
return failRestore(
|
|
3701
|
+
new AggregateError(
|
|
3702
|
+
[error, cleanupError],
|
|
3703
|
+
`Failed to discard a displaced ${application.name} writer directory during rollback`
|
|
3704
|
+
)
|
|
3705
|
+
);
|
|
3706
|
+
}
|
|
3707
|
+
} else {
|
|
3708
|
+
fallbackDisplacedPath = displacedPath;
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
restoreRetryDeadline ??= Date.now() + 5000;
|
|
3712
|
+
if (process.platform !== 'win32' && process.getuid?.() !== 0) {
|
|
3713
|
+
const stagedPlaceholderPath = join(
|
|
3714
|
+
asideStagingDir,
|
|
3715
|
+
`.rollback-placeholder-${process.pid}-${Date.now()}-${randomUUID()}`
|
|
3716
|
+
);
|
|
3717
|
+
try {
|
|
3718
|
+
if (asideIsSymbolicLink) {
|
|
3719
|
+
await writeFile(stagedPlaceholderPath, '', { flag: 'wx', mode: 0o000 });
|
|
3720
|
+
} else {
|
|
3721
|
+
await mkdir(stagedPlaceholderPath, { mode: 0o300 });
|
|
3722
|
+
}
|
|
3723
|
+
transactionPaths.add(stagedPlaceholderPath);
|
|
3724
|
+
let placeholderPlacementError: unknown;
|
|
3725
|
+
do {
|
|
3726
|
+
let writerDisplacedPath: string | undefined;
|
|
3727
|
+
try {
|
|
3728
|
+
writerDisplacedPath = displaceCurrentDirectorySync();
|
|
3729
|
+
renameSync(stagedPlaceholderPath, application.dirPath);
|
|
3730
|
+
if (!asideIsSymbolicLink) {
|
|
3731
|
+
try {
|
|
3732
|
+
chmodSync(application.dirPath, 0o100);
|
|
3733
|
+
} catch (chmodError) {
|
|
3734
|
+
try {
|
|
3735
|
+
renameSync(application.dirPath, stagedPlaceholderPath);
|
|
3736
|
+
} catch (compensationError) {
|
|
3737
|
+
throw new AggregateError(
|
|
3738
|
+
[chmodError, compensationError],
|
|
3739
|
+
`Failed to restrict and then restore the ${application.name} rollback placeholder`
|
|
3740
|
+
);
|
|
3741
|
+
}
|
|
3742
|
+
throw chmodError;
|
|
3743
|
+
}
|
|
3744
|
+
}
|
|
3745
|
+
transactionPaths.delete(stagedPlaceholderPath);
|
|
3746
|
+
placeholderPlacementError = undefined;
|
|
3747
|
+
} catch (placeholderError) {
|
|
3748
|
+
placeholderPlacementError = placeholderError;
|
|
3749
|
+
}
|
|
3750
|
+
if (writerDisplacedPath) {
|
|
3751
|
+
await rm(writerDisplacedPath, {
|
|
3752
|
+
recursive: true,
|
|
3753
|
+
force: true,
|
|
3754
|
+
maxRetries: 3,
|
|
3755
|
+
retryDelay: 100,
|
|
3756
|
+
});
|
|
3757
|
+
transactionPaths.delete(writerDisplacedPath);
|
|
3758
|
+
}
|
|
3759
|
+
if (
|
|
3760
|
+
placeholderPlacementError &&
|
|
3761
|
+
!retryableRenameCodes.has((placeholderPlacementError as NodeJS.ErrnoException).code ?? '')
|
|
3762
|
+
) {
|
|
3763
|
+
throw placeholderPlacementError;
|
|
3764
|
+
}
|
|
3765
|
+
if (!placeholderPlacementError) break;
|
|
3766
|
+
await delay(10);
|
|
3767
|
+
} while (Date.now() < restoreRetryDeadline);
|
|
3768
|
+
if (transactionPaths.has(stagedPlaceholderPath)) {
|
|
3769
|
+
throw new Error(
|
|
3770
|
+
`Failed to place the ${application.name} rollback placeholder before the deadline: ${errorMessage(placeholderPlacementError)}`,
|
|
3771
|
+
{ cause: placeholderPlacementError }
|
|
3772
|
+
);
|
|
3773
|
+
}
|
|
3774
|
+
const placeholder = await lstat(application.dirPath, { bigint: true });
|
|
3775
|
+
placeholderIdentity = { dev: placeholder.dev, ino: placeholder.ino };
|
|
3776
|
+
} catch (placeholderError) {
|
|
3777
|
+
return failRestore(
|
|
3778
|
+
new AggregateError(
|
|
3779
|
+
[error, placeholderError],
|
|
3780
|
+
`Failed to block a live ${application.name} writer during rollback`
|
|
3781
|
+
)
|
|
3782
|
+
);
|
|
3783
|
+
}
|
|
3784
|
+
}
|
|
849
3785
|
await delay(10);
|
|
850
3786
|
}
|
|
3787
|
+
} while (restoreRetryDeadline !== undefined && Date.now() < restoreRetryDeadline);
|
|
3788
|
+
return failRestore(restoreError);
|
|
3789
|
+
}
|
|
3790
|
+
try {
|
|
3791
|
+
await displaceCurrentDirectory();
|
|
3792
|
+
} catch (displaceError) {
|
|
3793
|
+
try {
|
|
3794
|
+
await rm(application.dirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
3795
|
+
await cleanupExtractionPaths(application, asideStagingDir, transactionPaths);
|
|
3796
|
+
return;
|
|
3797
|
+
} catch (removeError) {
|
|
3798
|
+
throw new AggregateError(
|
|
3799
|
+
[displaceError, removeError],
|
|
3800
|
+
`Failed to remove the partial ${application.name} component directory after extraction failed`
|
|
3801
|
+
);
|
|
851
3802
|
}
|
|
852
|
-
if (!restored) throw restoreError;
|
|
853
3803
|
}
|
|
3804
|
+
await cleanupExtractionPaths(application, asideStagingDir, transactionPaths);
|
|
3805
|
+
}
|
|
854
3806
|
|
|
855
|
-
|
|
3807
|
+
/**
|
|
3808
|
+
* The one definition of how Harper invokes a package manager to install dependencies; every npm
|
|
3809
|
+
* entry point composes its arguments here rather than assembling its own.
|
|
3810
|
+
*/
|
|
3811
|
+
export function packageManagerInstallArguments(
|
|
3812
|
+
packageManagerName: string,
|
|
3813
|
+
allowInstallScripts: boolean,
|
|
3814
|
+
force = false
|
|
3815
|
+
): string[] {
|
|
3816
|
+
const args = ['install'];
|
|
3817
|
+
if (force) args.push('--force');
|
|
3818
|
+
if (packageManagerName === 'npm') args.push('--omit=dev', '--no-audit', '--no-fund');
|
|
3819
|
+
if (!allowInstallScripts) args.push('--ignore-scripts');
|
|
3820
|
+
return args;
|
|
856
3821
|
}
|
|
857
3822
|
|
|
858
3823
|
/**
|
|
859
|
-
* Install
|
|
860
|
-
*
|
|
861
|
-
*
|
|
862
|
-
* package manager, `npm`.
|
|
3824
|
+
* Install a component's dependencies into `buildDirPath` — the live path, or a candidate under
|
|
3825
|
+
* `.deploy-staging`. Explicit rather than repointing `application.dirPath`, which is read after preparation
|
|
3826
|
+
* too and would name a vanished directory if any failure path skipped the restore.
|
|
863
3827
|
*
|
|
864
|
-
*
|
|
3828
|
+
* Uses a configured `application.install` command, a package manager derived from the application's
|
|
3829
|
+
* `package.json#devEngines`, or the default, `npm`. Returns early when `node_modules` already exists or
|
|
3830
|
+
* when the manifest has no automatic install work. An explicitly selected non-npm manager is always
|
|
3831
|
+
* allowed to inspect its own workspace configuration, even when the root manifest has no production
|
|
3832
|
+
* dependencies.
|
|
865
3833
|
*
|
|
866
|
-
*
|
|
3834
|
+
* May be called from any Harper thread as part of a serialized preparation.
|
|
867
3835
|
*/
|
|
868
|
-
export async function installApplication(application: Application) {
|
|
3836
|
+
export async function installApplication(application: Application, buildDirPath = application.dirPath) {
|
|
869
3837
|
let packageJSON: any;
|
|
870
3838
|
try {
|
|
871
|
-
packageJSON = JSON.parse(await readFile(join(
|
|
3839
|
+
packageJSON = JSON.parse(await readFile(join(buildDirPath, 'package.json'), 'utf8'));
|
|
872
3840
|
} catch (err) {
|
|
873
3841
|
if (err.code !== 'ENOENT') throw err;
|
|
874
3842
|
// If no package.json, nothing to install
|
|
@@ -877,7 +3845,7 @@ export async function installApplication(application: Application) {
|
|
|
877
3845
|
}
|
|
878
3846
|
try {
|
|
879
3847
|
// Does node_modules exist?
|
|
880
|
-
await access(join(
|
|
3848
|
+
await access(join(buildDirPath, 'node_modules'), constants.F_OK);
|
|
881
3849
|
application.logger.info(
|
|
882
3850
|
`Application ${application.name} already has node_modules; skipping install and treating the runtime as opaque for redeploy comparison`
|
|
883
3851
|
);
|
|
@@ -888,8 +3856,15 @@ export async function installApplication(application: Application) {
|
|
|
888
3856
|
// If node_modules doesn't exist, we need to install dependencies
|
|
889
3857
|
}
|
|
890
3858
|
|
|
3859
|
+
const allowInstallScripts = !!application.install?.allowInstallScripts;
|
|
3860
|
+
|
|
891
3861
|
// If custom install command is specified, run it
|
|
892
3862
|
if (application.install?.command) {
|
|
3863
|
+
if (application.install.allowInstallScripts === undefined) {
|
|
3864
|
+
application.logger.warn(
|
|
3865
|
+
`Application ${application.name} uses install_command without install_allow_scripts; package lifecycle scripts are disabled by default for npm and tools that honor npm_config_ignore_scripts, including npm run pre/post hooks. Set install_allow_scripts (or install.allowInstallScripts in root config) to true to opt in`
|
|
3866
|
+
);
|
|
3867
|
+
}
|
|
893
3868
|
const [command, ...args] = application.install.command.split(' ');
|
|
894
3869
|
const customOnLine = application.onInstallLine
|
|
895
3870
|
? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!(command, stream, line)
|
|
@@ -898,10 +3873,12 @@ export async function installApplication(application: Application) {
|
|
|
898
3873
|
application.name,
|
|
899
3874
|
command,
|
|
900
3875
|
args,
|
|
901
|
-
|
|
3876
|
+
buildDirPath,
|
|
902
3877
|
application.install?.timeout,
|
|
903
3878
|
customOnLine,
|
|
904
|
-
application.npmUserconfigPath
|
|
3879
|
+
application.npmUserconfigPath,
|
|
3880
|
+
undefined,
|
|
3881
|
+
!allowInstallScripts
|
|
905
3882
|
);
|
|
906
3883
|
// if it succeeds, return
|
|
907
3884
|
if (code === 0) {
|
|
@@ -921,9 +3898,21 @@ export async function installApplication(application: Application) {
|
|
|
921
3898
|
);
|
|
922
3899
|
}
|
|
923
3900
|
|
|
924
|
-
// Next, try package.json devEngines field
|
|
925
3901
|
const { packageManager } = packageJSON.devEngines || {};
|
|
3902
|
+
if (dependencyFieldHasWork(packageJSON, 'devDependencies')) {
|
|
3903
|
+
application.logger.warn(
|
|
3904
|
+
`Application ${application.name} declares devDependencies; automatic npm installation omits them, while explicitly selected non-npm package managers retain their own install defaults. Use install_command when deployment requires custom behavior`
|
|
3905
|
+
);
|
|
3906
|
+
}
|
|
3907
|
+
if (
|
|
3908
|
+
!packageHasAutomaticInstallWork(packageJSON) &&
|
|
3909
|
+
!(allowInstallScripts && packageHasAllowedInstallLifecycleWork(packageJSON))
|
|
3910
|
+
) {
|
|
3911
|
+
application.logger.info(`Application ${application.name} has no production package work; skipping install`);
|
|
3912
|
+
return;
|
|
3913
|
+
}
|
|
926
3914
|
|
|
3915
|
+
// Next, try package.json devEngines field
|
|
927
3916
|
// Custom package manager specified
|
|
928
3917
|
if (packageManager) {
|
|
929
3918
|
// On any given system we want to leverage the `name` to match the package manager executable
|
|
@@ -959,8 +3948,8 @@ export async function installApplication(application: Application) {
|
|
|
959
3948
|
const { stdout, stderr, code } = await nonInteractiveSpawn(
|
|
960
3949
|
application.name,
|
|
961
3950
|
(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name,
|
|
962
|
-
|
|
963
|
-
|
|
3951
|
+
packageManagerInstallArguments(packageManager.name, allowInstallScripts),
|
|
3952
|
+
buildDirPath,
|
|
964
3953
|
application.install?.timeout,
|
|
965
3954
|
pmOnLine,
|
|
966
3955
|
application.npmUserconfigPath
|
|
@@ -1008,9 +3997,13 @@ export async function installApplication(application: Application) {
|
|
|
1008
3997
|
}
|
|
1009
3998
|
|
|
1010
3999
|
// Finally, default to running `npm install`
|
|
1011
|
-
const npmInstallArgs =
|
|
1012
|
-
|
|
1013
|
-
|
|
4000
|
+
const npmInstallArgs = packageManagerInstallArguments('npm', allowInstallScripts, true);
|
|
4001
|
+
// A candidate build is installed at a staging path and then RENAMED to the live path, so nothing npm
|
|
4002
|
+
// writes may depend on the build location. npm links a `file:` dependency relatively on POSIX, which
|
|
4003
|
+
// survives the move — but as an absolute junction on Windows, which does not, so the dependency stops
|
|
4004
|
+
// resolving once the tree moves. `--install-links` copies instead of linking, leaving no path to break.
|
|
4005
|
+
// win32 and candidate builds only, since it does change how `file:` dependencies behave.
|
|
4006
|
+
if (process.platform === 'win32' && buildDirPath !== application.dirPath) npmInstallArgs.push('--install-links');
|
|
1014
4007
|
const npmOnLine = application.onInstallLine
|
|
1015
4008
|
? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!('npm', stream, line)
|
|
1016
4009
|
: undefined;
|
|
@@ -1018,7 +4011,7 @@ export async function installApplication(application: Application) {
|
|
|
1018
4011
|
application.name,
|
|
1019
4012
|
(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
|
|
1020
4013
|
npmInstallArgs,
|
|
1021
|
-
|
|
4014
|
+
buildDirPath,
|
|
1022
4015
|
application.install?.timeout,
|
|
1023
4016
|
npmOnLine,
|
|
1024
4017
|
application.npmUserconfigPath
|
|
@@ -1067,6 +4060,7 @@ export class Application {
|
|
|
1067
4060
|
name: string;
|
|
1068
4061
|
payload?: Buffer | string | Readable;
|
|
1069
4062
|
packageIdentifier?: string;
|
|
4063
|
+
readonly packLocalDirectory: boolean;
|
|
1070
4064
|
install?: { command?: string; timeout?: number; allowInstallScripts?: boolean };
|
|
1071
4065
|
onInstallLine?: OnInstallLine;
|
|
1072
4066
|
dirPath: string;
|
|
@@ -1092,6 +4086,7 @@ export class Application {
|
|
|
1092
4086
|
constructor({ name, payload, packageIdentifier, install, onInstallLine, credentials }: ApplicationOptions) {
|
|
1093
4087
|
this.name = name;
|
|
1094
4088
|
this.payload = payload;
|
|
4089
|
+
this.packLocalDirectory = shouldPackLocalDirectory(packageIdentifier);
|
|
1095
4090
|
this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
|
|
1096
4091
|
this.install = install;
|
|
1097
4092
|
this.onInstallLine = onInstallLine;
|
|
@@ -1202,7 +4197,12 @@ export class Application {
|
|
|
1202
4197
|
* during the installation process in order to actually resolve what the user specifies for a
|
|
1203
4198
|
* component matching some of npm's package resolution rules.
|
|
1204
4199
|
*/
|
|
4200
|
+
function isBareAbsolutePackagePath(packageIdentifier: string) {
|
|
4201
|
+
return isAbsolute(packageIdentifier) || win32.isAbsolute(packageIdentifier);
|
|
4202
|
+
}
|
|
4203
|
+
|
|
1205
4204
|
export function derivePackageIdentifier(packageIdentifier: string) {
|
|
4205
|
+
if (isBareAbsolutePackagePath(packageIdentifier)) return `file:${packageIdentifier}`;
|
|
1206
4206
|
if (packageIdentifier.includes(':')) {
|
|
1207
4207
|
return packageIdentifier;
|
|
1208
4208
|
}
|
|
@@ -1216,6 +4216,10 @@ export function derivePackageIdentifier(packageIdentifier: string) {
|
|
|
1216
4216
|
return `github:${packageIdentifier}`;
|
|
1217
4217
|
}
|
|
1218
4218
|
|
|
4219
|
+
export function shouldPackLocalDirectory(packageIdentifier: string | undefined, platform = process.platform) {
|
|
4220
|
+
return platform === 'win32' && !!packageIdentifier && isBareAbsolutePackagePath(packageIdentifier);
|
|
4221
|
+
}
|
|
4222
|
+
|
|
1219
4223
|
/**
|
|
1220
4224
|
* Extract and install the specified application.
|
|
1221
4225
|
*
|
|
@@ -1232,49 +4236,147 @@ export function derivePackageIdentifier(packageIdentifier: string) {
|
|
|
1232
4236
|
* @param application The application to prepare.
|
|
1233
4237
|
* @returns A promise that resolves when all preparation steps complete.
|
|
1234
4238
|
*/
|
|
1235
|
-
export
|
|
1236
|
-
|
|
4239
|
+
export type PrepareApplicationOptions = {
|
|
4240
|
+
beforePrepare?: () => Promise<void>;
|
|
4241
|
+
/**
|
|
4242
|
+
* Runs against the built candidate while the live version is still serving, and BEFORE the swap. A
|
|
4243
|
+
* throw here means the candidate never goes live — which is the whole difference from the previous
|
|
4244
|
+
* behavior, where the swap committed first and a load failure was reported over an already-live
|
|
4245
|
+
* broken release.
|
|
4246
|
+
*/
|
|
4247
|
+
validateCandidate?: (candidateDirPath: string) => Promise<void>;
|
|
4248
|
+
/**
|
|
4249
|
+
* Names `.deploy-staging/<artifactId>`, and is not the deploy-lifecycle token: that token de-duplicates
|
|
4250
|
+
* overlapping deploys of one component and is released by the first matching end, so two activations of
|
|
4251
|
+
* one artifact sharing it would resume watchers while the second is still swapping. Defaults to the
|
|
4252
|
+
* lifecycle token, which is unique per invocation.
|
|
4253
|
+
*/
|
|
4254
|
+
artifactId?: string;
|
|
4255
|
+
/**
|
|
4256
|
+
* `deploy` builds, certifies and activates. `stage` builds and certifies, then stops, leaving an
|
|
4257
|
+
* artifact a later `activate` can address by `artifactId`. `activate` verifies an existing artifact
|
|
4258
|
+
* and swaps it in, doing no resolution, no install and no network work at all.
|
|
4259
|
+
*/
|
|
4260
|
+
mode?: 'deploy' | 'stage' | 'activate';
|
|
4261
|
+
/**
|
|
4262
|
+
* `stage` only: the build's declared intent, recorded with the artifact for whoever activates it.
|
|
4263
|
+
* A callback rather than a value because `beforePrepare` is what determines it, and that runs after
|
|
4264
|
+
* these options have been constructed.
|
|
4265
|
+
*/
|
|
4266
|
+
describeArtifact?: () => { rootConfig: Record<string, unknown> | null; isolated: boolean };
|
|
4267
|
+
/**
|
|
4268
|
+
* `activate` only: publish the artifact's recorded root-config entry, immediately before the swap. May
|
|
4269
|
+
* return an undo, run if the activation then fails before it commits — otherwise a failed activation
|
|
4270
|
+
* leaves config naming a release that is not live, which the next boot install would resolve and build
|
|
4271
|
+
* from scratch over a component whose certified artifact is sitting beside it.
|
|
4272
|
+
*/
|
|
4273
|
+
publishRootConfig?: (entry: Record<string, unknown>) => Promise<(() => Promise<void>) | void>;
|
|
4274
|
+
/** `activate` only: admit the artifact's isolation intent under the preparation lock, before the swap. */
|
|
4275
|
+
admitIsolation?: (descriptor: ArtifactDescriptor) => Promise<void>;
|
|
4276
|
+
};
|
|
4277
|
+
|
|
4278
|
+
export async function prepareApplication(application: Application, options: PrepareApplicationOptions = {}) {
|
|
4279
|
+
const lifecycleToken = await broadcastDeployStart(application.name);
|
|
4280
|
+
const mode = options.mode ?? 'deploy';
|
|
4281
|
+
const artifactId = options.artifactId ?? lifecycleToken;
|
|
1237
4282
|
try {
|
|
1238
4283
|
const commandTimeoutMs = application.install?.timeout ?? DEFAULT_COMMAND_TIMEOUT_MS;
|
|
1239
4284
|
await withComponentPreparationLock(
|
|
1240
4285
|
application.dirPath,
|
|
1241
4286
|
async () => {
|
|
4287
|
+
await options.beforePrepare?.();
|
|
4288
|
+
const asideStagingDir = extractionStagingDirectory(application.dirPath);
|
|
4289
|
+
let recoveryPending = true;
|
|
4290
|
+
try {
|
|
4291
|
+
await lstat(asideStagingDir);
|
|
4292
|
+
} catch (error) {
|
|
4293
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
4294
|
+
recoveryPending = false;
|
|
4295
|
+
}
|
|
4296
|
+
// BEFORE the legacy pass. That pass refuses to restore while a journal survives, so skipping
|
|
4297
|
+
// this would not lose data — it would just stall the deploy behind its own unsettled state.
|
|
4298
|
+
//
|
|
4299
|
+
// The id this request names is pinned against the retention this runs. An activation would
|
|
4300
|
+
// otherwise have its own artifact deleted by its own preamble; and a redelivered stage would
|
|
4301
|
+
// have the artifact evicted out from under the exclusive claim below, which would then rebuild
|
|
4302
|
+
// different bytes under an id that already named some.
|
|
4303
|
+
await settleStagingForComponent(dirname(application.dirPath), application.name, artifactId);
|
|
4304
|
+
if (recoveryPending) {
|
|
4305
|
+
await ensureExtractionStagingDirectory(asideStagingDir);
|
|
4306
|
+
await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir);
|
|
4307
|
+
}
|
|
1242
4308
|
const previousPackageMetadata = await readInstalledPackageMetadata(application.dirPath);
|
|
1243
|
-
|
|
4309
|
+
// Determined before the swap, because both trees exist then: the runtime comparison below
|
|
4310
|
+
// wants the live version and the candidate side by side.
|
|
4311
|
+
application.isNewComponent = !(await lstat(application.dirPath).then(
|
|
4312
|
+
() => true,
|
|
4313
|
+
(error) => {
|
|
4314
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
|
4315
|
+
throw error;
|
|
4316
|
+
}
|
|
4317
|
+
));
|
|
4318
|
+
if (mode === 'activate') {
|
|
4319
|
+
await activateStagedArtifact(application, artifactId, previousPackageMetadata, options);
|
|
4320
|
+
return;
|
|
4321
|
+
}
|
|
1244
4322
|
try {
|
|
1245
|
-
// Materialize the per-deploy `.npmrc` before
|
|
1246
|
-
//
|
|
4323
|
+
// Materialize the per-deploy `.npmrc` before the build so both `npm pack` and `npm install`
|
|
4324
|
+
// authenticate against the private registry; always remove it afterward.
|
|
1247
4325
|
await application.writeTransientNpmrc();
|
|
4326
|
+
let candidateDirPath: string;
|
|
1248
4327
|
try {
|
|
1249
|
-
//
|
|
1250
|
-
//
|
|
1251
|
-
//
|
|
1252
|
-
// allowed to run) executes.
|
|
4328
|
+
// Backstop only: the builder closes the session as soon as extraction is done, so the
|
|
4329
|
+
// credential is already gone before any install script runs. This finally covers the paths
|
|
4330
|
+
// that fail before it gets there.
|
|
1253
4331
|
await application.startGitCredentialSession();
|
|
1254
|
-
|
|
4332
|
+
candidateDirPath = await buildCandidateApplication(application, artifactId, {
|
|
4333
|
+
rejectLinkSource: mode === 'stage',
|
|
4334
|
+
});
|
|
1255
4335
|
} finally {
|
|
1256
4336
|
await application.cleanupGitCredentialSession();
|
|
1257
4337
|
}
|
|
1258
|
-
await installApplication(application);
|
|
1259
|
-
if (!application.isNewComponent) {
|
|
1260
|
-
const currentPackageMetadata = await readInstalledPackageMetadata(application.dirPath);
|
|
1261
|
-
application.packageMetadataChanged = installedRuntimeChanged(
|
|
1262
|
-
previousPackageMetadata,
|
|
1263
|
-
currentPackageMetadata,
|
|
1264
|
-
application.installationIsOpaque
|
|
1265
|
-
);
|
|
1266
|
-
}
|
|
1267
|
-
await extraction?.commit();
|
|
1268
|
-
} catch (error) {
|
|
1269
4338
|
try {
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
4339
|
+
// Validated while the previous version is still the one serving, so a candidate that
|
|
4340
|
+
// installs cleanly but throws at load is rejected without ever having been live.
|
|
4341
|
+
await options.validateCandidate?.(candidateDirPath);
|
|
4342
|
+
if (!application.isNewComponent) {
|
|
4343
|
+
application.packageMetadataChanged = installedRuntimeChanged(
|
|
4344
|
+
previousPackageMetadata,
|
|
4345
|
+
await readInstalledPackageMetadata(candidateDirPath),
|
|
4346
|
+
application.installationIsOpaque
|
|
4347
|
+
);
|
|
4348
|
+
}
|
|
4349
|
+
if (mode === 'stage') {
|
|
4350
|
+
await assertOwnedArtifactTree(candidateDirPath, application.name);
|
|
4351
|
+
// The descriptor goes first so `.complete` vouches for it: after this pair the artifact
|
|
4352
|
+
// is dormant, and a delayed activation reads its build's decisions from here because
|
|
4353
|
+
// nothing on disk carries them otherwise.
|
|
4354
|
+
const declared = options.describeArtifact?.() ?? { rootConfig: null, isolated: false };
|
|
4355
|
+
await writeArtifactDescriptor(application.dirPath, artifactId, {
|
|
4356
|
+
v: ARTIFACT_DESCRIPTOR_VERSION,
|
|
4357
|
+
component: application.name,
|
|
4358
|
+
rootConfig: declared.rootConfig,
|
|
4359
|
+
installationIsOpaque: application.installationIsOpaque,
|
|
4360
|
+
isolated: declared.isolated,
|
|
4361
|
+
});
|
|
4362
|
+
await markCandidateComplete(application.dirPath, artifactId, application.name);
|
|
4363
|
+
await syncArtifactAncestors(candidateDeploymentDirPath(application.dirPath, artifactId));
|
|
4364
|
+
return;
|
|
4365
|
+
}
|
|
4366
|
+
await markCandidateComplete(application.dirPath, artifactId, application.name);
|
|
4367
|
+
await activateCandidateApplication(application, artifactId);
|
|
4368
|
+
} catch (error) {
|
|
4369
|
+
// The builder's own cleanup only covers a failed BUILD. A rejected validation, or an
|
|
4370
|
+
// activation that was cleanly compensated, would otherwise leave a whole installed
|
|
4371
|
+
// dependency tree under this deployment id — repeated rejections fill the volume.
|
|
4372
|
+
//
|
|
4373
|
+
// NOT when compensation itself failed. There the previous version is not back and the live
|
|
4374
|
+
// path may be absent, and the candidate plus its `.complete` marker and journal are exactly
|
|
4375
|
+
// what recovery needs to roll the validated deploy forward at the next start. Discarding
|
|
4376
|
+
// them there trades a bounded disk cost for a component with no version at all.
|
|
4377
|
+
if (!compensationIncomplete(error)) await discardCandidate(application, artifactId);
|
|
4378
|
+
throw error;
|
|
1276
4379
|
}
|
|
1277
|
-
throw error;
|
|
1278
4380
|
} finally {
|
|
1279
4381
|
await application.cleanupTransientNpmrc();
|
|
1280
4382
|
}
|
|
@@ -1301,10 +4403,84 @@ export async function prepareApplication(application: Application) {
|
|
|
1301
4403
|
}
|
|
1302
4404
|
);
|
|
1303
4405
|
} finally {
|
|
1304
|
-
broadcastDeployEnd(application.name,
|
|
4406
|
+
broadcastDeployEnd(application.name, lifecycleToken);
|
|
1305
4407
|
}
|
|
1306
4408
|
}
|
|
1307
4409
|
|
|
4410
|
+
/**
|
|
4411
|
+
* Swap an already-certified artifact into the live path. Called with the component's preparation lock held
|
|
4412
|
+
* and after the same recovery preamble every build runs, so the state read here is settled.
|
|
4413
|
+
*
|
|
4414
|
+
* Nothing is resolved, fetched or installed: the bytes were certified when they were staged, which is the
|
|
4415
|
+
* whole point of addressing one by id. Verification is therefore the only gate, and it is strict — this is
|
|
4416
|
+
* activation input for a build this process did not make and may not have made on this node.
|
|
4417
|
+
*
|
|
4418
|
+
* A rejection here must NOT discard: the artifact belongs to whoever staged it, a wrong-component request
|
|
4419
|
+
* must not delete another component's build, and a retry needs what a failed attempt left behind.
|
|
4420
|
+
*/
|
|
4421
|
+
async function activateStagedArtifact(
|
|
4422
|
+
application: Application,
|
|
4423
|
+
artifactId: string,
|
|
4424
|
+
previousPackageMetadata: InstalledPackageMetadata,
|
|
4425
|
+
options: PrepareApplicationOptions
|
|
4426
|
+
): Promise<void> {
|
|
4427
|
+
const deploymentDirPath = candidateDeploymentDirPath(application.dirPath, artifactId);
|
|
4428
|
+
const candidateDirPath = candidateApplicationPath(application.dirPath, artifactId);
|
|
4429
|
+
// 404 means only one thing — no artifact answers to this id here — so a caller can tell "never existed or
|
|
4430
|
+
// already used" from "present, but not something this can activate", which is every other refusal below
|
|
4431
|
+
// and a 409 like the rest of that family.
|
|
4432
|
+
const refuse = (why: string, statusCode: number) =>
|
|
4433
|
+
new ClientError(`Cannot deploy ${application.name} from deployment ${artifactId}: ${why}`, statusCode);
|
|
4434
|
+
const missing = (why: string) => refuse(why, 404);
|
|
4435
|
+
const unusable = (why: string) => refuse(why, 409);
|
|
4436
|
+
|
|
4437
|
+
const owner = await candidateComponentName(deploymentDirPath);
|
|
4438
|
+
if (owner === undefined) throw missing('there is no staged build with that id on this node');
|
|
4439
|
+
if (owner !== application.name) throw unusable(`that staged build belongs to '${owner}'`);
|
|
4440
|
+
if (!(await presentOrAbsent(join(deploymentDirPath, CANDIDATE_COMPLETE_MARKER)))) {
|
|
4441
|
+
throw unusable('its build never completed');
|
|
4442
|
+
}
|
|
4443
|
+
if (await presentOrAbsent(join(deploymentDirPath, UNSETTLED_MARKER))) {
|
|
4444
|
+
throw unusable('recovery could not settle it, so it is not safe to activate');
|
|
4445
|
+
}
|
|
4446
|
+
if (await presentOrAbsent(join(deploymentDirPath, ACTIVATION_JOURNAL))) {
|
|
4447
|
+
throw unusable('an activation of it is unsettled');
|
|
4448
|
+
}
|
|
4449
|
+
const candidateStat = await presentOrAbsent(candidateDirPath);
|
|
4450
|
+
if (!candidateStat || !candidateStat.isDirectory()) {
|
|
4451
|
+
// Unlike an immediate deploy, a symlink is refused: a `file:` directory is linked rather than
|
|
4452
|
+
// copied, so what it points at now is not what was certified. Staging rejects the source for the
|
|
4453
|
+
// same reason; this is the other end of the same rule, for an artifact staged by an older build.
|
|
4454
|
+
throw unusable('its build tree is missing or is a link rather than a copy');
|
|
4455
|
+
}
|
|
4456
|
+
const descriptor = await readArtifactDescriptor(deploymentDirPath, application.name);
|
|
4457
|
+
if (!descriptor) throw unusable('it does not record what its build decided');
|
|
4458
|
+
|
|
4459
|
+
await options.admitIsolation?.(descriptor);
|
|
4460
|
+
|
|
4461
|
+
// `.complete` is a durability marker over the bytes, not a seal on them, and every check that made this
|
|
4462
|
+
// artifact safe ran at stage time. The link rule especially cannot be skipped:
|
|
4463
|
+
// `repairRelocatedDependencyLinks` runs past the commit point and can only warn, so a link planted while
|
|
4464
|
+
// the artifact sat dormant would otherwise go live with the operation reporting success.
|
|
4465
|
+
await assertOwnedArtifactTree(candidateDirPath, application.name, 'activate');
|
|
4466
|
+
// Also the activation's `prepare` phase end: the deploy path emits `prepare`/`start` for every mode but
|
|
4467
|
+
// only ever emitted its `done` from here.
|
|
4468
|
+
await options.validateCandidate?.(candidateDirPath);
|
|
4469
|
+
|
|
4470
|
+
if (!application.isNewComponent) {
|
|
4471
|
+
application.packageMetadataChanged = installedRuntimeChanged(
|
|
4472
|
+
previousPackageMetadata,
|
|
4473
|
+
await readInstalledPackageMetadata(candidateDirPath),
|
|
4474
|
+
descriptor.installationIsOpaque
|
|
4475
|
+
);
|
|
4476
|
+
}
|
|
4477
|
+
await activateCandidateApplication(application, artifactId, {
|
|
4478
|
+
// Returns its own undo, which the swap runs inside its pre-commit boundary — see there for why it
|
|
4479
|
+
// cannot be run out here.
|
|
4480
|
+
afterJournal: descriptor.rootConfig ? () => options.publishRootConfig!(descriptor.rootConfig!) : undefined,
|
|
4481
|
+
});
|
|
4482
|
+
}
|
|
4483
|
+
|
|
1308
4484
|
/**
|
|
1309
4485
|
* Install all applications specified in the root config.
|
|
1310
4486
|
*
|
|
@@ -1629,19 +4805,6 @@ export function buildNpmrcContent(registryCredentials: ResolvedRegistryCredentia
|
|
|
1629
4805
|
return lines.join('\n') + '\n';
|
|
1630
4806
|
}
|
|
1631
4807
|
|
|
1632
|
-
/**
|
|
1633
|
-
* Execute a command (using `spawn`) with stdin ignored.
|
|
1634
|
-
*
|
|
1635
|
-
* Stdout is logged chunk-by-chunk. Stderr is buffered and then logged line-by-line.
|
|
1636
|
-
*
|
|
1637
|
-
* Rejects with an error if the command fails or times out.
|
|
1638
|
-
*
|
|
1639
|
-
* @param command The command to run.
|
|
1640
|
-
* @param args The arguments to pass to the command.
|
|
1641
|
-
* @param cwd The working directory for the command.
|
|
1642
|
-
* @param timeoutMs The timeout for the command in milliseconds. Defaults to 5 minutes.
|
|
1643
|
-
* @returns A promise that resolves when the command completes.
|
|
1644
|
-
*/
|
|
1645
4808
|
/**
|
|
1646
4809
|
* Line-buffered split that emits complete `\n`-terminated lines as they
|
|
1647
4810
|
* arrive, holding any partial trailing fragment until the next chunk or `flush()`.
|
|
@@ -1683,9 +4846,20 @@ function createLineSplitter(onLine: (line: string) => void): {
|
|
|
1683
4846
|
}
|
|
1684
4847
|
|
|
1685
4848
|
/**
|
|
1686
|
-
*
|
|
1687
|
-
* spawn. The keys are decrypted to a transient 0700 dir
|
|
1688
|
-
* soon as the process settles — on success, failure, and
|
|
4849
|
+
* Execute a command (using `spawn`) with stdin ignored, with the deploy's SSH key material
|
|
4850
|
+
* materialized only for the duration of the spawn. The keys are decrypted to a transient 0700 dir
|
|
4851
|
+
* (see `materializeGitSSH`) and removed as soon as the process settles — on success, failure, and
|
|
4852
|
+
* timeout alike.
|
|
4853
|
+
*
|
|
4854
|
+
* Stdout is logged chunk-by-chunk. Stderr is buffered and then logged line-by-line.
|
|
4855
|
+
*
|
|
4856
|
+
* Rejects with an error if the command fails or times out.
|
|
4857
|
+
*
|
|
4858
|
+
* @param command The command to run.
|
|
4859
|
+
* @param args The arguments to pass to the command.
|
|
4860
|
+
* @param cwd The working directory for the command.
|
|
4861
|
+
* @param timeoutMs The timeout for the command in milliseconds. Defaults to DEFAULT_COMMAND_TIMEOUT_MS.
|
|
4862
|
+
* @returns A promise that resolves when the command completes.
|
|
1689
4863
|
*/
|
|
1690
4864
|
export async function nonInteractiveSpawn(
|
|
1691
4865
|
applicationName: string,
|
|
@@ -1695,7 +4869,8 @@ export async function nonInteractiveSpawn(
|
|
|
1695
4869
|
timeoutMs: number = DEFAULT_COMMAND_TIMEOUT_MS,
|
|
1696
4870
|
onLine?: (stream: 'stdout' | 'stderr', line: string) => void,
|
|
1697
4871
|
npmUserconfigPath?: string,
|
|
1698
|
-
gitCredentialEnv?: Record<string, string
|
|
4872
|
+
gitCredentialEnv?: Record<string, string>,
|
|
4873
|
+
ignoreNpmScripts = false
|
|
1699
4874
|
): Promise<{ stdout: string; stderr: string; code: number }> {
|
|
1700
4875
|
const gitSSH = await materializeGitSSH();
|
|
1701
4876
|
try {
|
|
@@ -1708,7 +4883,8 @@ export async function nonInteractiveSpawn(
|
|
|
1708
4883
|
onLine,
|
|
1709
4884
|
npmUserconfigPath,
|
|
1710
4885
|
gitSSH?.command,
|
|
1711
|
-
gitCredentialEnv
|
|
4886
|
+
gitCredentialEnv,
|
|
4887
|
+
ignoreNpmScripts
|
|
1712
4888
|
);
|
|
1713
4889
|
} finally {
|
|
1714
4890
|
await gitSSH?.cleanup();
|
|
@@ -1724,7 +4900,8 @@ function spawnWithEnv(
|
|
|
1724
4900
|
onLine: ((stream: 'stdout' | 'stderr', line: string) => void) | undefined,
|
|
1725
4901
|
npmUserconfigPath: string | undefined,
|
|
1726
4902
|
gitSSHCommand: string | undefined,
|
|
1727
|
-
gitCredentialEnv: Record<string, string> | undefined
|
|
4903
|
+
gitCredentialEnv: Record<string, string> | undefined,
|
|
4904
|
+
ignoreNpmScripts: boolean
|
|
1728
4905
|
): Promise<{ stdout: string; stderr: string; code: number }> {
|
|
1729
4906
|
return new Promise((resolve, reject) => {
|
|
1730
4907
|
logger
|
|
@@ -1760,10 +4937,17 @@ function spawnWithEnv(
|
|
|
1760
4937
|
}
|
|
1761
4938
|
env.npm_config_userconfig = npmUserconfigPath;
|
|
1762
4939
|
}
|
|
4940
|
+
if (ignoreNpmScripts) {
|
|
4941
|
+
for (const key of Object.keys(env)) {
|
|
4942
|
+
if (key.toLowerCase() === 'npm_config_ignore_scripts') delete env[key];
|
|
4943
|
+
}
|
|
4944
|
+
env.npm_config_ignore_scripts = 'true';
|
|
4945
|
+
}
|
|
1763
4946
|
|
|
1764
4947
|
if (process.platform === 'win32' && command === 'npm') {
|
|
1765
4948
|
command = 'npm.cmd';
|
|
1766
4949
|
}
|
|
4950
|
+
const spawnLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}`);
|
|
1767
4951
|
|
|
1768
4952
|
const childProcess = spawn(command, args, {
|
|
1769
4953
|
shell: true,
|
|
@@ -1857,7 +5041,22 @@ function spawnWithEnv(
|
|
|
1857
5041
|
}
|
|
1858
5042
|
});
|
|
1859
5043
|
|
|
1860
|
-
childProcess.on('
|
|
5044
|
+
childProcess.on('exit', (code, signal) => {
|
|
5045
|
+
spawnLogger.debug?.(`Direct child exited with code ${code}, signal ${signal}; awaiting stdio close`);
|
|
5046
|
+
});
|
|
5047
|
+
|
|
5048
|
+
childProcess.on('close', async (code, signal) => {
|
|
5049
|
+
if (didTimeout) {
|
|
5050
|
+
spawnLogger.debug?.(
|
|
5051
|
+
`Child stdio closed with code ${code}, signal ${signal}; timeout path owns process-tree confirmation`
|
|
5052
|
+
);
|
|
5053
|
+
} else if (trackedProcessId) {
|
|
5054
|
+
spawnLogger.debug?.(
|
|
5055
|
+
`Child stdio closed with code ${code}, signal ${signal}; confirming process-tree termination`
|
|
5056
|
+
);
|
|
5057
|
+
} else {
|
|
5058
|
+
spawnLogger.debug?.(`Child stdio closed with code ${code}, signal ${signal}; no process tree was tracked`);
|
|
5059
|
+
}
|
|
1861
5060
|
resolveClose();
|
|
1862
5061
|
clearTimeout(timeout);
|
|
1863
5062
|
// A successful direct-child exit does not prove the process group is empty: a custom
|
|
@@ -1878,6 +5077,7 @@ function spawnWithEnv(
|
|
|
1878
5077
|
}
|
|
1879
5078
|
return;
|
|
1880
5079
|
}
|
|
5080
|
+
spawnLogger.debug?.(`Process tree termination confirmed after command close with code ${code}`);
|
|
1881
5081
|
untrackProcessGroup();
|
|
1882
5082
|
}
|
|
1883
5083
|
// When didTimeout is true, the timeout path's own terminateProcessTree(...).then(...) owns
|
|
@@ -1889,7 +5089,6 @@ function spawnWithEnv(
|
|
|
1889
5089
|
if (stderr) {
|
|
1890
5090
|
printStd(applicationName, command, stderr, 'stderr');
|
|
1891
5091
|
}
|
|
1892
|
-
logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug?.(`Process exited with code ${code}`);
|
|
1893
5092
|
if (didTimeout || didSettle) return;
|
|
1894
5093
|
didSettle = true;
|
|
1895
5094
|
resolve({
|
|
@@ -1914,12 +5113,7 @@ class CommandTimeoutError extends Error {
|
|
|
1914
5113
|
}
|
|
1915
5114
|
|
|
1916
5115
|
function processGroupIsAlive(processGroupId: number): boolean {
|
|
1917
|
-
|
|
1918
|
-
process.kill(-processGroupId, 0);
|
|
1919
|
-
return true;
|
|
1920
|
-
} catch (error: any) {
|
|
1921
|
-
return error.code === 'EPERM';
|
|
1922
|
-
}
|
|
5116
|
+
return isProcessGroupAlive(processGroupId);
|
|
1923
5117
|
}
|
|
1924
5118
|
|
|
1925
5119
|
async function waitForProcessGroupExit(processGroupId: number, timeoutMs: number): Promise<boolean> {
|