@harperfast/harper 5.2.13 → 5.3.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (655) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +58 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +120 -4
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +3436 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +440 -98
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +56 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +381 -32
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/deploymentOperations.ts +4 -1
  23. package/components/deploymentRecorder.ts +9 -2
  24. package/components/mcp/adapters/harperHttp.ts +4 -0
  25. package/components/mcp/listChanged.ts +4 -0
  26. package/components/mcp/toolRegistry.ts +2 -0
  27. package/components/mcp/tools/operations.ts +9 -0
  28. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  29. package/components/operations.js +537 -113
  30. package/components/operationsValidation.js +98 -3
  31. package/components/packageComponent.ts +25 -1
  32. package/components/requestRestart.ts +11 -0
  33. package/components/status/ComponentStatusRegistry.ts +59 -0
  34. package/config/RootConfigWatcher.ts +240 -40
  35. package/config/configReadRetry.ts +62 -0
  36. package/config/configUtils.ts +357 -48
  37. package/config/harperConfigEnvVars.ts +170 -27
  38. package/config/parseConfigFile.ts +34 -0
  39. package/config/readConfigFileSync.ts +44 -0
  40. package/config/watcherArming.ts +59 -0
  41. package/config-root.schema.json +33 -0
  42. package/dataLayer/blobBackup.ts +160 -50
  43. package/dataLayer/delete.ts +6 -1
  44. package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
  45. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  46. package/dataLayer/hdbInfoController.ts +34 -1
  47. package/dataLayer/insert.ts +44 -1
  48. package/dataLayer/rocksdbBackup.ts +53 -10
  49. package/dataLayer/schema.ts +11 -1
  50. package/dataLayer/schemaDescribe.ts +8 -1
  51. package/dist/agent/mcpTools.js +1 -1
  52. package/dist/agent/mcpTools.js.map +1 -1
  53. package/dist/agent/session.d.ts +22 -0
  54. package/dist/agent/session.js +26 -15
  55. package/dist/agent/session.js.map +1 -1
  56. package/dist/bin/cliOperations.js +61 -9
  57. package/dist/bin/cliOperations.js.map +1 -1
  58. package/dist/bin/copyDb.d.ts +12 -1
  59. package/dist/bin/copyDb.js +248 -60
  60. package/dist/bin/copyDb.js.map +1 -1
  61. package/dist/bin/deploySetup.d.ts +2 -0
  62. package/dist/bin/deploySetup.js +11 -3
  63. package/dist/bin/deploySetup.js.map +1 -1
  64. package/dist/bin/harper.js +1 -1
  65. package/dist/bin/harper.js.map +1 -1
  66. package/dist/bin/help.js +4 -1
  67. package/dist/bin/help.js.map +1 -1
  68. package/dist/bin/lite.js +4 -1
  69. package/dist/bin/lite.js.map +1 -1
  70. package/dist/bin/restart.js +91 -6
  71. package/dist/bin/restart.js.map +1 -1
  72. package/dist/bin/run.js +4 -10
  73. package/dist/bin/run.js.map +1 -1
  74. package/dist/bin/upgrade.js +4 -3
  75. package/dist/bin/upgrade.js.map +1 -1
  76. package/dist/bin/workloadIdentity.d.ts +18 -0
  77. package/dist/bin/workloadIdentity.js +100 -0
  78. package/dist/bin/workloadIdentity.js.map +1 -0
  79. package/dist/components/Application.d.ts +235 -17
  80. package/dist/components/Application.js +3038 -238
  81. package/dist/components/Application.js.map +1 -1
  82. package/dist/components/ApplicationScope.d.ts +8 -0
  83. package/dist/components/ApplicationScope.js +7 -0
  84. package/dist/components/ApplicationScope.js.map +1 -1
  85. package/dist/components/EntryHandler.js +26 -10
  86. package/dist/components/EntryHandler.js.map +1 -1
  87. package/dist/components/OptionsWatcher.d.ts +4 -0
  88. package/dist/components/OptionsWatcher.js +440 -99
  89. package/dist/components/OptionsWatcher.js.map +1 -1
  90. package/dist/components/RuntimeModuleTracker.js +40 -6
  91. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  92. package/dist/components/Scope.js +52 -13
  93. package/dist/components/Scope.js.map +1 -1
  94. package/dist/components/awaitRestart.d.ts +33 -0
  95. package/dist/components/awaitRestart.js +61 -0
  96. package/dist/components/awaitRestart.js.map +1 -0
  97. package/dist/components/componentLoader.d.ts +38 -1
  98. package/dist/components/componentLoader.js +313 -24
  99. package/dist/components/componentLoader.js.map +1 -1
  100. package/dist/components/componentPreparationLock.d.ts +5 -0
  101. package/dist/components/componentPreparationLock.js +14 -6
  102. package/dist/components/componentPreparationLock.js.map +1 -1
  103. package/dist/components/deploymentOperations.js +4 -1
  104. package/dist/components/deploymentOperations.js.map +1 -1
  105. package/dist/components/deploymentRecorder.d.ts +4 -2
  106. package/dist/components/deploymentRecorder.js +1 -0
  107. package/dist/components/deploymentRecorder.js.map +1 -1
  108. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  109. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  110. package/dist/components/mcp/listChanged.js +5 -0
  111. package/dist/components/mcp/listChanged.js.map +1 -1
  112. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  113. package/dist/components/mcp/toolRegistry.js.map +1 -1
  114. package/dist/components/mcp/tools/operations.d.ts +5 -0
  115. package/dist/components/mcp/tools/operations.js +9 -0
  116. package/dist/components/mcp/tools/operations.js.map +1 -1
  117. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  118. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  119. package/dist/components/operations.d.ts +28 -0
  120. package/dist/components/operations.js +476 -113
  121. package/dist/components/operations.js.map +1 -1
  122. package/dist/components/operationsValidation.js +97 -3
  123. package/dist/components/operationsValidation.js.map +1 -1
  124. package/dist/components/packageComponent.js +24 -0
  125. package/dist/components/packageComponent.js.map +1 -1
  126. package/dist/components/requestRestart.d.ts +1 -0
  127. package/dist/components/requestRestart.js +7 -0
  128. package/dist/components/requestRestart.js.map +1 -1
  129. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  130. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  131. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  132. package/dist/config/RootConfigWatcher.d.ts +9 -1
  133. package/dist/config/RootConfigWatcher.js +228 -37
  134. package/dist/config/RootConfigWatcher.js.map +1 -1
  135. package/dist/config/configReadRetry.d.ts +8 -0
  136. package/dist/config/configReadRetry.js +62 -0
  137. package/dist/config/configReadRetry.js.map +1 -0
  138. package/dist/config/configUtils.d.ts +11 -2
  139. package/dist/config/configUtils.js +304 -47
  140. package/dist/config/configUtils.js.map +1 -1
  141. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  142. package/dist/config/harperConfigEnvVars.js +162 -25
  143. package/dist/config/harperConfigEnvVars.js.map +1 -1
  144. package/dist/config/parseConfigFile.d.ts +4 -0
  145. package/dist/config/parseConfigFile.js +35 -0
  146. package/dist/config/parseConfigFile.js.map +1 -0
  147. package/dist/config/readConfigFileSync.d.ts +1 -0
  148. package/dist/config/readConfigFileSync.js +47 -0
  149. package/dist/config/readConfigFileSync.js.map +1 -0
  150. package/dist/config/watcherArming.d.ts +15 -0
  151. package/dist/config/watcherArming.js +59 -0
  152. package/dist/config/watcherArming.js.map +1 -0
  153. package/dist/dataLayer/blobBackup.d.ts +49 -20
  154. package/dist/dataLayer/blobBackup.js +139 -50
  155. package/dist/dataLayer/blobBackup.js.map +1 -1
  156. package/dist/dataLayer/delete.js +1 -1
  157. package/dist/dataLayer/delete.js.map +1 -1
  158. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  159. package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
  160. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  161. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  162. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  163. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  164. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  165. package/dist/dataLayer/hdbInfoController.js +30 -1
  166. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  167. package/dist/dataLayer/insert.d.ts +9 -1
  168. package/dist/dataLayer/insert.js +30 -0
  169. package/dist/dataLayer/insert.js.map +1 -1
  170. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  171. package/dist/dataLayer/rocksdbBackup.js +45 -8
  172. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  173. package/dist/dataLayer/schema.js +8 -0
  174. package/dist/dataLayer/schema.js.map +1 -1
  175. package/dist/dataLayer/schemaDescribe.js +8 -1
  176. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  177. package/dist/index.d.ts +2 -1
  178. package/dist/index.js +4 -1
  179. package/dist/index.js.map +1 -1
  180. package/dist/json/systemSchema.json +55 -0
  181. package/dist/resources/DatabaseTransaction.d.ts +77 -0
  182. package/dist/resources/DatabaseTransaction.js +586 -54
  183. package/dist/resources/DatabaseTransaction.js.map +1 -1
  184. package/dist/resources/LMDBTransaction.d.ts +2 -1
  185. package/dist/resources/LMDBTransaction.js +43 -6
  186. package/dist/resources/LMDBTransaction.js.map +1 -1
  187. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  188. package/dist/resources/PrimaryRocksDatabase.js +49 -5
  189. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  190. package/dist/resources/RecordEncoder.d.ts +20 -0
  191. package/dist/resources/RecordEncoder.js +110 -9
  192. package/dist/resources/RecordEncoder.js.map +1 -1
  193. package/dist/resources/RequestTarget.d.ts +2 -0
  194. package/dist/resources/RequestTarget.js.map +1 -1
  195. package/dist/resources/Resource.js +117 -24
  196. package/dist/resources/Resource.js.map +1 -1
  197. package/dist/resources/ResourceInterface.d.ts +28 -1
  198. package/dist/resources/ResourceInterface.js.map +1 -1
  199. package/dist/resources/RocksIndexStore.d.ts +6 -1
  200. package/dist/resources/RocksIndexStore.js +24 -9
  201. package/dist/resources/RocksIndexStore.js.map +1 -1
  202. package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
  203. package/dist/resources/RocksTransactionLogStore.js +160 -49
  204. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  205. package/dist/resources/Table.d.ts +165 -10
  206. package/dist/resources/Table.js +1805 -303
  207. package/dist/resources/Table.js.map +1 -1
  208. package/dist/resources/analytics/write.js +10 -3
  209. package/dist/resources/analytics/write.js.map +1 -1
  210. package/dist/resources/auditStore.d.ts +181 -2
  211. package/dist/resources/auditStore.js +640 -29
  212. package/dist/resources/auditStore.js.map +1 -1
  213. package/dist/resources/blob.d.ts +129 -9
  214. package/dist/resources/blob.js +992 -114
  215. package/dist/resources/blob.js.map +1 -1
  216. package/dist/resources/branchDatabase.d.ts +48 -0
  217. package/dist/resources/branchDatabase.js +892 -0
  218. package/dist/resources/branchDatabase.js.map +1 -0
  219. package/dist/resources/crdt.js +50 -12
  220. package/dist/resources/crdt.js.map +1 -1
  221. package/dist/resources/dataLoader.js +3 -4
  222. package/dist/resources/dataLoader.js.map +1 -1
  223. package/dist/resources/databases.d.ts +157 -14
  224. package/dist/resources/databases.js +1462 -325
  225. package/dist/resources/databases.js.map +1 -1
  226. package/dist/resources/defineTable.d.ts +10 -2
  227. package/dist/resources/defineTable.js +9 -1
  228. package/dist/resources/defineTable.js.map +1 -1
  229. package/dist/resources/derivedIndexRegistry.d.ts +5 -0
  230. package/dist/resources/derivedIndexRegistry.js +68 -0
  231. package/dist/resources/derivedIndexRegistry.js.map +1 -0
  232. package/dist/resources/derivedIndexRuntime.d.ts +215 -0
  233. package/dist/resources/derivedIndexRuntime.js +2027 -0
  234. package/dist/resources/derivedIndexRuntime.js.map +1 -0
  235. package/dist/resources/graphql.d.ts +1 -1
  236. package/dist/resources/graphql.js +52 -16
  237. package/dist/resources/graphql.js.map +1 -1
  238. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
  239. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
  240. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  241. package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
  242. package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
  243. package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
  244. package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
  245. package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
  246. package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
  247. package/dist/resources/longLivedTransactions.d.ts +71 -0
  248. package/dist/resources/longLivedTransactions.js +358 -0
  249. package/dist/resources/longLivedTransactions.js.map +1 -0
  250. package/dist/resources/models/backendRegistry.d.ts +26 -0
  251. package/dist/resources/models/backendRegistry.js +60 -2
  252. package/dist/resources/models/backendRegistry.js.map +1 -1
  253. package/dist/resources/models/bootstrap.d.ts +33 -1
  254. package/dist/resources/models/bootstrap.js +416 -31
  255. package/dist/resources/models/bootstrap.js.map +1 -1
  256. package/dist/resources/nodeIdMapping.d.ts +5 -0
  257. package/dist/resources/nodeIdMapping.js +49 -0
  258. package/dist/resources/nodeIdMapping.js.map +1 -1
  259. package/dist/resources/recordLock.d.ts +123 -0
  260. package/dist/resources/recordLock.js +315 -0
  261. package/dist/resources/recordLock.js.map +1 -0
  262. package/dist/resources/recordLockCoordinator.d.ts +557 -0
  263. package/dist/resources/recordLockCoordinator.js +2565 -0
  264. package/dist/resources/recordLockCoordinator.js.map +1 -0
  265. package/dist/resources/replayLogs.d.ts +14 -1
  266. package/dist/resources/replayLogs.js +157 -24
  267. package/dist/resources/replayLogs.js.map +1 -1
  268. package/dist/resources/replayLogsGuards.d.ts +94 -7
  269. package/dist/resources/replayLogsGuards.js +111 -7
  270. package/dist/resources/replayLogsGuards.js.map +1 -1
  271. package/dist/resources/replicatedApplyFailure.d.ts +16 -0
  272. package/dist/resources/replicatedApplyFailure.js +63 -0
  273. package/dist/resources/replicatedApplyFailure.js.map +1 -0
  274. package/dist/resources/scheduler/scheduler.js +3 -3
  275. package/dist/resources/scheduler/scheduler.js.map +1 -1
  276. package/dist/resources/search.d.ts +10 -4
  277. package/dist/resources/search.js +283 -37
  278. package/dist/resources/search.js.map +1 -1
  279. package/dist/resources/tracked.d.ts +5 -1
  280. package/dist/resources/tracked.js +74 -23
  281. package/dist/resources/tracked.js.map +1 -1
  282. package/dist/resources/transactionBroadcast.d.ts +1 -1
  283. package/dist/resources/transactionBroadcast.js +2 -2
  284. package/dist/resources/transactionBroadcast.js.map +1 -1
  285. package/dist/security/auth.js +61 -30
  286. package/dist/security/auth.js.map +1 -1
  287. package/dist/security/authn/oidc/claims.d.ts +22 -0
  288. package/dist/security/authn/oidc/claims.js +71 -0
  289. package/dist/security/authn/oidc/claims.js.map +1 -0
  290. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  291. package/dist/security/authn/oidc/identityToken.js +111 -0
  292. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  293. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  294. package/dist/security/authn/oidc/jwks.js +261 -0
  295. package/dist/security/authn/oidc/jwks.js.map +1 -0
  296. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  297. package/dist/security/authn/oidc/providers/generic.js +34 -0
  298. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  299. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  300. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  301. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  302. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  303. package/dist/security/authn/oidc/providers/index.js +24 -0
  304. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  305. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  306. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  307. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  308. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  309. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  310. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  311. package/dist/security/authn/oidc/types.d.ts +38 -0
  312. package/dist/security/authn/oidc/types.js +6 -0
  313. package/dist/security/authn/oidc/types.js.map +1 -0
  314. package/dist/security/certificateVerification/index.js +40 -11
  315. package/dist/security/certificateVerification/index.js.map +1 -1
  316. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  317. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  318. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  319. package/dist/security/certificateVerification/types.d.ts +1 -0
  320. package/dist/security/credentialProvenance.d.ts +35 -0
  321. package/dist/security/credentialProvenance.js +51 -0
  322. package/dist/security/credentialProvenance.js.map +1 -0
  323. package/dist/security/credentialRejection.d.ts +4 -0
  324. package/dist/security/credentialRejection.js +24 -0
  325. package/dist/security/credentialRejection.js.map +1 -0
  326. package/dist/security/deferredAuthentication.d.ts +36 -0
  327. package/dist/security/deferredAuthentication.js +70 -0
  328. package/dist/security/deferredAuthentication.js.map +1 -0
  329. package/dist/security/impersonation.d.ts +21 -0
  330. package/dist/security/impersonation.js +108 -9
  331. package/dist/security/impersonation.js.map +1 -1
  332. package/dist/security/jsLoader.d.ts +6 -0
  333. package/dist/security/jsLoader.js +77 -15
  334. package/dist/security/jsLoader.js.map +1 -1
  335. package/dist/security/keys.js +301 -71
  336. package/dist/security/keys.js.map +1 -1
  337. package/dist/security/operationScope.d.ts +21 -0
  338. package/dist/security/operationScope.js +36 -0
  339. package/dist/security/operationScope.js.map +1 -0
  340. package/dist/security/permissionsTranslator.js +21 -0
  341. package/dist/security/permissionsTranslator.js.map +1 -1
  342. package/dist/security/tokenAuthentication.d.ts +19 -1
  343. package/dist/security/tokenAuthentication.js +191 -10
  344. package/dist/security/tokenAuthentication.js.map +1 -1
  345. package/dist/security/user.js +4 -3
  346. package/dist/security/user.js.map +1 -1
  347. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  348. package/dist/server/DurableSubscriptionsSession.js +67 -10
  349. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  350. package/dist/server/REST.js +106 -2
  351. package/dist/server/REST.js.map +1 -1
  352. package/dist/server/graphqlQuerying.js +4 -0
  353. package/dist/server/graphqlQuerying.js.map +1 -1
  354. package/dist/server/http.d.ts +16 -1
  355. package/dist/server/http.js +122 -17
  356. package/dist/server/http.js.map +1 -1
  357. package/dist/server/itc/serverHandlers.js +8 -1
  358. package/dist/server/itc/serverHandlers.js.map +1 -1
  359. package/dist/server/jobs/jobProcess.js +6 -2
  360. package/dist/server/jobs/jobProcess.js.map +1 -1
  361. package/dist/server/jobs/jobs.js +4 -1
  362. package/dist/server/jobs/jobs.js.map +1 -1
  363. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  364. package/dist/server/liveSubscriptionAuth.js +105 -39
  365. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  366. package/dist/server/loadRootComponents.js +49 -10
  367. package/dist/server/loadRootComponents.js.map +1 -1
  368. package/dist/server/mqtt.d.ts +2 -0
  369. package/dist/server/mqtt.js +165 -30
  370. package/dist/server/mqtt.js.map +1 -1
  371. package/dist/server/nodeName.d.ts +2 -0
  372. package/dist/server/nodeName.js +107 -23
  373. package/dist/server/nodeName.js.map +1 -1
  374. package/dist/server/serverHelpers/Headers.d.ts +27 -0
  375. package/dist/server/serverHelpers/Headers.js +145 -1
  376. package/dist/server/serverHelpers/Headers.js.map +1 -1
  377. package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
  378. package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
  379. package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
  380. package/dist/server/serverHelpers/Request.d.ts +5 -10
  381. package/dist/server/serverHelpers/Request.js +38 -136
  382. package/dist/server/serverHelpers/Request.js.map +1 -1
  383. package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
  384. package/dist/server/serverHelpers/contentTypes.js +221 -40
  385. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  386. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  387. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  388. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  389. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  390. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  391. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  392. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  393. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  394. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  395. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  396. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  397. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  398. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  399. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  400. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  401. package/dist/server/serverHelpers/serverUtilities.js +178 -29
  402. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  403. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  404. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  405. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  406. package/dist/server/serverHelpers/uwsServer.js +19 -1
  407. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  408. package/dist/server/static.js +24 -28
  409. package/dist/server/static.js.map +1 -1
  410. package/dist/server/storageReclamation.d.ts +5 -0
  411. package/dist/server/storageReclamation.js +17 -1
  412. package/dist/server/storageReclamation.js.map +1 -1
  413. package/dist/server/threads/isolatedApplications.d.ts +47 -0
  414. package/dist/server/threads/isolatedApplications.js +171 -0
  415. package/dist/server/threads/isolatedApplications.js.map +1 -0
  416. package/dist/server/threads/itc.d.ts +7 -2
  417. package/dist/server/threads/itc.js +5 -1
  418. package/dist/server/threads/itc.js.map +1 -1
  419. package/dist/server/threads/logRotationTransport.d.ts +1 -0
  420. package/dist/server/threads/logRotationTransport.js +33 -0
  421. package/dist/server/threads/logRotationTransport.js.map +1 -0
  422. package/dist/server/threads/manageThreads.d.ts +83 -3
  423. package/dist/server/threads/manageThreads.js +769 -75
  424. package/dist/server/threads/manageThreads.js.map +1 -1
  425. package/dist/server/threads/socketRouter.d.ts +1 -0
  426. package/dist/server/threads/socketRouter.js +277 -31
  427. package/dist/server/threads/socketRouter.js.map +1 -1
  428. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  429. package/dist/server/threads/threadHeapMemory.js +31 -0
  430. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  431. package/dist/server/threads/threadServer.js +76 -22
  432. package/dist/server/threads/threadServer.js.map +1 -1
  433. package/dist/sqlEngine/config.d.ts +1 -3
  434. package/dist/sqlEngine/config.js +19 -16
  435. package/dist/sqlEngine/config.js.map +1 -1
  436. package/dist/sqlTranslator/index.d.ts +1 -1
  437. package/dist/sqlTranslator/index.js +30 -7
  438. package/dist/sqlTranslator/index.js.map +1 -1
  439. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  440. package/dist/upgrade/directives/5-3-0.js +148 -0
  441. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  442. package/dist/upgrade/directives/directivesController.js +2 -1
  443. package/dist/upgrade/directives/directivesController.js.map +1 -1
  444. package/dist/utility/OperationFunctionCaller.js +2 -1
  445. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  446. package/dist/utility/common_utils.d.ts +16 -0
  447. package/dist/utility/common_utils.js +32 -6
  448. package/dist/utility/common_utils.js.map +1 -1
  449. package/dist/utility/componentNames.d.ts +8 -0
  450. package/dist/utility/componentNames.js +12 -1
  451. package/dist/utility/componentNames.js.map +1 -1
  452. package/dist/utility/environment/environmentManager.js +3 -6
  453. package/dist/utility/environment/environmentManager.js.map +1 -1
  454. package/dist/utility/environment/systemInformation.d.ts +1 -0
  455. package/dist/utility/environment/systemInformation.js +1 -0
  456. package/dist/utility/environment/systemInformation.js.map +1 -1
  457. package/dist/utility/errors/commonErrors.d.ts +2 -0
  458. package/dist/utility/errors/commonErrors.js +2 -0
  459. package/dist/utility/errors/commonErrors.js.map +1 -1
  460. package/dist/utility/errors/hdbError.d.ts +33 -0
  461. package/dist/utility/errors/hdbError.js +58 -1
  462. package/dist/utility/errors/hdbError.js.map +1 -1
  463. package/dist/utility/globalSchema.d.ts +18 -0
  464. package/dist/utility/hdbTerms.d.ts +18 -0
  465. package/dist/utility/hdbTerms.js +20 -2
  466. package/dist/utility/hdbTerms.js.map +1 -1
  467. package/dist/utility/logging/harper_logger.d.ts +2 -0
  468. package/dist/utility/logging/harper_logger.js +286 -30
  469. package/dist/utility/logging/harper_logger.js.map +1 -1
  470. package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
  471. package/dist/utility/logging/logGenerationCoordinator.js +184 -0
  472. package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
  473. package/dist/utility/logging/logRotation.d.ts +46 -0
  474. package/dist/utility/logging/logRotation.js +365 -0
  475. package/dist/utility/logging/logRotation.js.map +1 -0
  476. package/dist/utility/logging/logRotator.d.ts +1 -1
  477. package/dist/utility/logging/logRotator.js +192 -85
  478. package/dist/utility/logging/logRotator.js.map +1 -1
  479. package/dist/utility/nodeIdentity.d.ts +9 -0
  480. package/dist/utility/nodeIdentity.js +58 -0
  481. package/dist/utility/nodeIdentity.js.map +1 -0
  482. package/dist/utility/npmUtilities.js +11 -7
  483. package/dist/utility/npmUtilities.js.map +1 -1
  484. package/dist/utility/operationPermissions.d.ts +3 -1
  485. package/dist/utility/operationPermissions.js +16 -1
  486. package/dist/utility/operationPermissions.js.map +1 -1
  487. package/dist/utility/operation_authorization.d.ts +10 -7
  488. package/dist/utility/operation_authorization.js +212 -41
  489. package/dist/utility/operation_authorization.js.map +1 -1
  490. package/dist/utility/watchPath.d.ts +29 -0
  491. package/dist/utility/watchPath.js +68 -0
  492. package/dist/utility/watchPath.js.map +1 -0
  493. package/dist/utility/watcherFallback.d.ts +41 -0
  494. package/dist/utility/watcherFallback.js +153 -0
  495. package/dist/utility/watcherFallback.js.map +1 -1
  496. package/dist/validation/configValidator.d.ts +12 -0
  497. package/dist/validation/configValidator.js +205 -75
  498. package/dist/validation/configValidator.js.map +1 -1
  499. package/dist/validation/installValidator.js +12 -0
  500. package/dist/validation/installValidator.js.map +1 -1
  501. package/dist/validation/validationWrapper.d.ts +11 -0
  502. package/dist/validation/validationWrapper.js +16 -3
  503. package/dist/validation/validationWrapper.js.map +1 -1
  504. package/index.ts +8 -0
  505. package/json/systemSchema.json +55 -0
  506. package/npm-shrinkwrap.json +286 -194
  507. package/package.json +15 -7
  508. package/resources/DESIGN.md +219 -52
  509. package/resources/DatabaseTransaction.ts +657 -52
  510. package/resources/LMDBTransaction.ts +46 -6
  511. package/resources/PrimaryRocksDatabase.ts +46 -7
  512. package/resources/RecordEncoder.ts +124 -9
  513. package/resources/RequestTarget.ts +2 -0
  514. package/resources/Resource.ts +114 -22
  515. package/resources/ResourceInterface.ts +31 -0
  516. package/resources/RocksIndexStore.ts +30 -9
  517. package/resources/RocksTransactionLogStore.ts +188 -51
  518. package/resources/Table.ts +1960 -301
  519. package/resources/analytics/write.ts +10 -3
  520. package/resources/auditStore.ts +647 -32
  521. package/resources/blob.ts +1029 -110
  522. package/resources/branchDatabase.ts +941 -0
  523. package/resources/crdt.ts +76 -12
  524. package/resources/dataLoader.ts +3 -4
  525. package/resources/databases.ts +1584 -320
  526. package/resources/defineTable.ts +18 -2
  527. package/resources/derivedIndexRegistry.ts +56 -0
  528. package/resources/derivedIndexRuntime.ts +2292 -0
  529. package/resources/graphql.ts +72 -17
  530. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
  531. package/resources/indexes/hnswDerivedIndex.ts +531 -0
  532. package/resources/indexes/hnswPlaneBinding.ts +174 -0
  533. package/resources/longLivedTransactions.ts +360 -0
  534. package/resources/models/backendRegistry.ts +84 -2
  535. package/resources/models/bootstrap.ts +473 -28
  536. package/resources/nodeIdMapping.ts +50 -0
  537. package/resources/recordLock.ts +419 -0
  538. package/resources/recordLockCoordinator.ts +3043 -0
  539. package/resources/replayLogs.ts +152 -26
  540. package/resources/replayLogsGuards.ts +171 -8
  541. package/resources/replicatedApplyFailure.ts +77 -0
  542. package/resources/scheduler/scheduler.ts +4 -4
  543. package/resources/search.ts +288 -46
  544. package/resources/tracked.ts +73 -22
  545. package/resources/transactionBroadcast.ts +3 -3
  546. package/security/auth.ts +68 -29
  547. package/security/authn/oidc/claims.ts +72 -0
  548. package/security/authn/oidc/identityToken.ts +129 -0
  549. package/security/authn/oidc/jwks.ts +260 -0
  550. package/security/authn/oidc/providers/generic.ts +40 -0
  551. package/security/authn/oidc/providers/githubActions.ts +137 -0
  552. package/security/authn/oidc/providers/index.ts +52 -0
  553. package/security/authn/oidc/tokenExchange.ts +300 -0
  554. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  555. package/security/authn/oidc/types.ts +41 -0
  556. package/security/certificateVerification/index.ts +54 -13
  557. package/security/certificateVerification/trustedIssuers.ts +76 -0
  558. package/security/certificateVerification/types.ts +1 -0
  559. package/security/credentialProvenance.ts +47 -0
  560. package/security/credentialRejection.ts +22 -0
  561. package/security/deferredAuthentication.ts +71 -0
  562. package/security/impersonation.ts +117 -12
  563. package/security/jsLoader.ts +83 -18
  564. package/security/keys.ts +298 -72
  565. package/security/operationScope.ts +33 -0
  566. package/security/permissionsTranslator.js +23 -0
  567. package/security/tokenAuthentication.ts +233 -12
  568. package/security/user.ts +4 -3
  569. package/server/DESIGN.md +194 -16
  570. package/server/DurableSubscriptionsSession.ts +71 -11
  571. package/server/REST.ts +115 -4
  572. package/server/graphqlQuerying.ts +4 -0
  573. package/server/http.ts +133 -20
  574. package/server/itc/serverHandlers.js +8 -1
  575. package/server/jobs/jobProcess.ts +8 -2
  576. package/server/jobs/jobs.ts +4 -1
  577. package/server/liveSubscriptionAuth.ts +129 -46
  578. package/server/loadRootComponents.js +50 -8
  579. package/server/mqtt.ts +179 -38
  580. package/server/nodeName.ts +103 -21
  581. package/server/serverHelpers/Headers.ts +135 -0
  582. package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
  583. package/server/serverHelpers/Request.ts +33 -131
  584. package/server/serverHelpers/contentTypes.ts +217 -36
  585. package/server/serverHelpers/deployValidationState.ts +24 -13
  586. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  587. package/server/serverHelpers/registeredOperations.ts +79 -22
  588. package/server/serverHelpers/requestSanitization.ts +15 -0
  589. package/server/serverHelpers/serverHandlers.js +6 -3
  590. package/server/serverHelpers/serverUtilities.ts +232 -40
  591. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  592. package/server/serverHelpers/uwsServer.ts +17 -2
  593. package/server/static.ts +23 -29
  594. package/server/storageReclamation.ts +15 -2
  595. package/server/threads/isolatedApplications.ts +157 -0
  596. package/server/threads/itc.js +11 -1
  597. package/server/threads/logRotationTransport.ts +40 -0
  598. package/server/threads/manageThreads.js +771 -66
  599. package/server/threads/socketRouter.ts +291 -30
  600. package/server/threads/threadHeapMemory.ts +26 -0
  601. package/server/threads/threadServer.js +73 -22
  602. package/sqlTranslator/index.ts +31 -8
  603. package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
  604. package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
  605. package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
  606. package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
  607. package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
  608. package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
  609. package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
  610. package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
  611. package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
  612. package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
  613. package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
  614. package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
  615. package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
  616. package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
  617. package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
  618. package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
  619. package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
  620. package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
  621. package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
  622. package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
  623. package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
  624. package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
  625. package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
  626. package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
  627. package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
  628. package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
  629. package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
  630. package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
  631. package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
  632. package/studio/web/index.html +1 -1
  633. package/upgrade/directives/5-3-0.ts +132 -0
  634. package/upgrade/directives/directivesController.ts +2 -1
  635. package/utility/OperationFunctionCaller.ts +2 -1
  636. package/utility/common_utils.ts +30 -5
  637. package/utility/componentNames.ts +12 -0
  638. package/utility/environment/environmentManager.ts +3 -7
  639. package/utility/environment/systemInformation.ts +7 -0
  640. package/utility/errors/commonErrors.ts +4 -0
  641. package/utility/errors/hdbError.ts +57 -0
  642. package/utility/hdbTerms.ts +19 -0
  643. package/utility/logging/harper_logger.ts +278 -26
  644. package/utility/logging/logGenerationCoordinator.ts +196 -0
  645. package/utility/logging/logRotation.ts +367 -0
  646. package/utility/logging/logRotator.ts +213 -81
  647. package/utility/nodeIdentity.ts +45 -0
  648. package/utility/npmUtilities.ts +12 -8
  649. package/utility/operationPermissions.ts +18 -1
  650. package/utility/operation_authorization.ts +231 -42
  651. package/utility/watchPath.ts +63 -0
  652. package/utility/watcherFallback.ts +148 -0
  653. package/validation/configValidator.ts +215 -75
  654. package/validation/installValidator.ts +15 -0
  655. package/validation/validationWrapper.ts +18 -4
@@ -0,0 +1,196 @@
1
+ 'use strict';
2
+
3
+ // Every isolate holds its own descriptor on the shared log file, so a rename leaves the others
4
+ // appending to the archived inode and destroying it loses whatever they write next. No size
5
+ // comparison can rule out a write that has not happened yet; enumerated in-process peers must answer.
6
+ //
7
+ // The thread layer injects the transport; harper_logger must never reach manageThreads.
8
+
9
+ import { statSync } from 'node:fs';
10
+ import { resolve } from 'node:path';
11
+ import { isMainThread } from 'node:worker_threads';
12
+
13
+ export const LOG_GENERATION_ROTATED = 'log_generation_rotated';
14
+ export const LOG_GENERATION_CLOSED = 'log_generation_closed';
15
+
16
+ // Under manageThreads' 30s ITC default, so an unanswered generation is decided here (retain the
17
+ // plain archive) rather than by a transport backstop that cannot express "unproven".
18
+ const DEFAULT_QUIESCENCE_TIMEOUT = 15000;
19
+ // A tiny maxSize can rotate faster than a slow peer answers. Overflow is left as a plain archive
20
+ // for the audit sweep instead of retaining one timer and one mesh request per generation.
21
+ const MAX_PENDING_RELEASE_REQUESTS = 64;
22
+
23
+ interface RotationTransport {
24
+ broadcast(message: any): void;
25
+ sendToThread(threadId: number, message: any): void;
26
+ onMessage(type: string, handler: (message: any) => void): void;
27
+ onThreadExit(handler: (threadId: number) => void): void;
28
+ peerThreadIds(): Iterable<number>;
29
+ threadId: number;
30
+ quiescenceTimeout?: number;
31
+ }
32
+
33
+ let transport: RotationTransport | undefined;
34
+ type LogSink = { identity(): any; close(): void };
35
+ // A set per path, not one sink: harper_logger caches its file loggers by the raw configured path, so
36
+ // two spellings of one file are two sinks holding two descriptors on it, and a release has to close
37
+ // both. The key is resolved because retention compares resolved paths.
38
+ const sinksByPath = new Map<string, Set<LogSink>>();
39
+ const pendingByRequest = new Map<
40
+ string,
41
+ { expected: Set<number>; liveLogPaths: Set<string>; settle: (released: boolean) => void }
42
+ >();
43
+ let requestCounter = 0;
44
+
45
+ export function setRotationTransport(newTransport?: RotationTransport) {
46
+ transport = newTransport;
47
+ if (!newTransport) return;
48
+ newTransport.onMessage(LOG_GENERATION_ROTATED, (message) => {
49
+ releaseLocally(message);
50
+ newTransport.sendToThread(message.originator, {
51
+ type: LOG_GENERATION_CLOSED,
52
+ request: message.request,
53
+ threadId: newTransport.threadId,
54
+ // The paths this isolate writes. Components load in the workers, so a component's own log
55
+ // is registered only there, and the thread that runs retention would otherwise see a live
56
+ // file it has never heard of and delete it by age.
57
+ logPaths: message.stale ? [...sinksByPath.keys()] : undefined,
58
+ });
59
+ });
60
+ newTransport.onMessage(LOG_GENERATION_CLOSED, (message) => {
61
+ recordPeerResponse(message.request, message.threadId, message.logPaths);
62
+ });
63
+ newTransport.onThreadExit((threadId) => {
64
+ // An exited thread cannot hold a descriptor, so its exit answers everything it owed.
65
+ for (const request of [...pendingByRequest.keys()]) recordPeerResponse(request, threadId);
66
+ });
67
+ }
68
+
69
+ /**
70
+ * Registered by the file sink, not the size guard: a thread with no guard still holds a descriptor,
71
+ * and its answer has to mean the descriptor is gone.
72
+ */
73
+ export function registerLogSink(logPath: string, sink: LogSink) {
74
+ const key = resolve(logPath);
75
+ const sinks = sinksByPath.get(key) ?? new Set<LogSink>();
76
+ sinks.add(sink);
77
+ sinksByPath.set(key, sinks);
78
+ }
79
+
80
+ export function unregisterLogSink(logPath: string, sink?: LogSink) {
81
+ const key = resolve(logPath);
82
+ if (!sink) return void sinksByPath.delete(key);
83
+ const sinks = sinksByPath.get(key);
84
+ if (!sinks?.delete(sink)) return;
85
+ if (sinks.size === 0) sinksByPath.delete(key);
86
+ }
87
+
88
+ export function nextGenerationId() {
89
+ return `${process.pid}-${transport?.threadId ?? 0}-${requestCounter++}`;
90
+ }
91
+
92
+ /**
93
+ * Ask every in-process peer this isolate can enumerate to release the descriptors this request
94
+ * names. `false` means the caller must leave those archives in place — it is never safe to unlink
95
+ * an inode a peer may still be appending to.
96
+ */
97
+ function requestRelease(message: any, deadline?: number): Promise<{ released: boolean; liveLogPaths: Set<string> }> {
98
+ releaseLocally(message);
99
+ const liveLogPaths = new Set<string>(sinksByPath.keys());
100
+ // No transport in a worker isolate means the mesh cannot be enumerated yet, not that there is
101
+ // nobody to ask: harper_logger installs the size guard at its own module load and manageThreads
102
+ // installs the transport at the end of its, so a worker rotating in that window would otherwise
103
+ // call an empty peer set a proof and unlink the generation its siblings are appending to. On the
104
+ // main thread the same state means no worker has been spawned, where zero peers is the truth.
105
+ if (!transport) return Promise.resolve({ released: isMainThread, liveLogPaths });
106
+ const expected = new Set<number>(transport.peerThreadIds());
107
+ if (expected.size === 0) return Promise.resolve({ released: true, liveLogPaths });
108
+ if (pendingByRequest.size >= MAX_PENDING_RELEASE_REQUESTS) return Promise.resolve({ released: false, liveLogPaths });
109
+ const quiescenceTimeout = transport.quiescenceTimeout ?? DEFAULT_QUIESCENCE_TIMEOUT;
110
+ if (deadline !== undefined && Date.now() + quiescenceTimeout > deadline)
111
+ return Promise.resolve({ released: false, liveLogPaths });
112
+ const request = message.request;
113
+ return new Promise((resolve) => {
114
+ let timer;
115
+ const settle = (released) => {
116
+ clearTimeout(timer);
117
+ pendingByRequest.delete(request);
118
+ resolve({ released, liveLogPaths });
119
+ };
120
+ pendingByRequest.set(request, { expected, liveLogPaths, settle });
121
+ timer = setTimeout(() => settle(false), quiescenceTimeout);
122
+ timer.unref?.();
123
+ transport.broadcast({ ...message, originator: transport.threadId });
124
+ });
125
+ }
126
+
127
+ /** One archived generation: release any descriptor still pointing at the inode that was renamed. */
128
+ export async function requestGenerationClose(generation: any, deadline?: number): Promise<boolean> {
129
+ return (
130
+ await requestRelease(
131
+ {
132
+ type: LOG_GENERATION_ROTATED,
133
+ request: generation.generation,
134
+ logPath: generation.logPath,
135
+ ino: generation.ino,
136
+ dev: generation.dev,
137
+ },
138
+ deadline
139
+ )
140
+ ).released;
141
+ }
142
+
143
+ /**
144
+ * Every generation but the live one, for every log this process writes, in a single round trip.
145
+ * Not scoped to one path: the archive directory holds the archives of every component and external
146
+ * log sharing it, and retention destroys those too.
147
+ */
148
+ export function requestStaleDescriptorRelease(
149
+ deadline?: number
150
+ ): Promise<{ released: boolean; liveLogPaths: Set<string> }> {
151
+ return requestRelease({ type: LOG_GENERATION_ROTATED, request: nextGenerationId(), stale: true }, deadline);
152
+ }
153
+
154
+ function recordPeerResponse(request: string, threadId: number, logPaths?: string[]) {
155
+ const pending = pendingByRequest.get(request);
156
+ if (!pending) return;
157
+ if (logPaths) for (const logPath of logPaths) pending.liveLogPaths.add(logPath);
158
+ pending.expected.delete(threadId);
159
+ if (pending.expected.size === 0) pending.settle(true);
160
+ }
161
+
162
+ function releaseLocally(message: any) {
163
+ if (message.stale) return releaseStaleDescriptors();
164
+ for (const sink of sinksByPath.get(resolve(message.logPath)) ?? []) {
165
+ const identity = sink.identity();
166
+ // No identity to compare (no descriptor open, or a filesystem that cannot report one) means
167
+ // closing is the only answer that can still be called a release.
168
+ if (!identity || !identity.ino || !message.ino || (identity.ino === message.ino && identity.dev === message.dev))
169
+ sink.close();
170
+ }
171
+ }
172
+
173
+ /**
174
+ * Release every descriptor this isolate holds that is not on the live generation of its own path.
175
+ * Each sink decides for itself, so nothing has to name the live inode of a log it does not own.
176
+ */
177
+ function releaseStaleDescriptors() {
178
+ for (const [logPath, sinks] of sinksByPath) {
179
+ let live;
180
+ try {
181
+ live = statSync(logPath);
182
+ } catch {
183
+ for (const sink of sinks) sink.close();
184
+ continue;
185
+ }
186
+ for (const sink of sinks) {
187
+ const identity = sink.identity();
188
+ // No identity is not the same as no descriptor: openLogFile() leaves the descriptor open when
189
+ // its fstat fails. Skipping the sink and still answering "released" is what lets an archive be
190
+ // unlinked under it, so close it here as the per-generation path already does.
191
+ // `ino === 0` (some Windows filesystems) proves nothing for the same reason.
192
+ if (!identity || !identity.ino || !live.ino || identity.ino !== live.ino || identity.dev !== live.dev)
193
+ sink.close();
194
+ }
195
+ }
196
+ }
@@ -0,0 +1,367 @@
1
+ 'use strict';
2
+
3
+ // The rotation primitives the log write path needs. Kept free of every Harper import so
4
+ // harper_logger.ts can pull it in eagerly: logRotator.ts reaches server/storageReclamation.ts ->
5
+ // manageThreads.js -> harper_logger.ts, which is why it may only ever be required lazily, and a
6
+ // guard installed on a timer is a guard the first megabytes of a burst are written without.
7
+
8
+ import {
9
+ createReadStream,
10
+ createWriteStream,
11
+ existsSync,
12
+ mkdirSync,
13
+ renameSync,
14
+ statSync,
15
+ promises as fsProm,
16
+ } from 'node:fs';
17
+ import { createGzip } from 'node:zlib';
18
+ import { pipeline } from 'node:stream/promises';
19
+ import { basename, dirname, extname, join, resolve } from 'node:path';
20
+ import { createHash } from 'node:crypto';
21
+ import { isMainThread, threadId } from 'node:worker_threads';
22
+ import { nextGenerationId, requestGenerationClose } from './logGenerationCoordinator.ts';
23
+
24
+ // Bounds each writer's blind window at one quantum plus the flush that crosses it, so the file is
25
+ // bounded by maxBytes + T * (quantum + batch) rather than by elapsed time. Fixed, not a remaining
26
+ // budget: a budget assumes one thread's stat accounts for the other threads' future bytes.
27
+ const CHECK_QUANTUM_DIVISOR = 16;
28
+ const ROTATION_RETRY_COOLDOWN = 5000;
29
+ const SIZE_UNIT_MULTIPLIERS = { K: 1e3, M: 1e6, G: 1e9 };
30
+
31
+ export const INVALID_MAX_SIZE_MSG = "'maxSize' must be a positive size with a K, M or G unit (for example '64M')";
32
+
33
+ /**
34
+ * Convert `logging.rotation.maxSize` to bytes, or undefined if it cannot be a byte limit. One
35
+ * definition, shared with validation/configValidator.ts.
36
+ */
37
+ export function parseMaxSize(maxSize: any) {
38
+ if (typeof maxSize !== 'string') return undefined;
39
+ const multiplier = SIZE_UNIT_MULTIPLIERS[maxSize.slice(-1)];
40
+ if (!multiplier) return undefined;
41
+ const size = maxSize.slice(0, -1);
42
+ // Number(), not a stricter grammar, so every mantissa that yields a usable cap today keeps working.
43
+ if (size.trim() === '') return undefined;
44
+ const bytes = Number(size) * multiplier;
45
+ return Number.isSafeInteger(bytes) && bytes > 0 ? bytes : undefined;
46
+ }
47
+
48
+ export function resolveRotatedLogDir(logPath: string, configuredPath?: string) {
49
+ return configuredPath || join(dirname(logPath), 'rotated');
50
+ }
51
+
52
+ // threadId is load-bearing in the suffix below: worker threads share the process pid, and this
53
+ // counter is per-isolate, so pid+seq alone cannot keep two threads' archives apart.
54
+ let rotationSequence = 0;
55
+
56
+ // A configured rotation path may also contain live logs, so destructive scans must identify archives.
57
+ const ARCHIVE_NAME = /-[0-9a-f]{8}-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{3}Z-\d+-\d+-\d+\.log(\.gz)?$/;
58
+
59
+ export function isArchiveName(file: string) {
60
+ return ARCHIVE_NAME.test(file);
61
+ }
62
+
63
+ export function archivePathFor(logPath: string, rotatedLogDir: string) {
64
+ // Name the archive after its source log (hdb, external, a component name, ...), not a fixed
65
+ // "HDB" literal — external/component loggers inherit rotation from the main logger (#1877) and
66
+ // default to the same rotated directory. A basename alone is not enough either: two distinct
67
+ // source paths can share one (`/logs/a/hdb.log`, `/logs/b/hdb.log`), so a hash of the resolved
68
+ // source path plus the unique suffix give every archive a name rename() can never clobber.
69
+ const sourceName = basename(logPath, extname(logPath)) || 'HDB';
70
+ // sha256, not sha1: this only needs a stable identifier, but a FIPS-mode OpenSSL provider
71
+ // disables sha1 and throws synchronously, which would crash every rotation.
72
+ const sourceId = createHash('sha256').update(resolve(logPath)).digest('hex').slice(0, 8);
73
+ const uniqueSuffix = `${process.pid}-${threadId}-${rotationSequence++}`;
74
+ const timestamp = new Date().toISOString().replaceAll(':', '-');
75
+ return join(rotatedLogDir, `${sourceName}-${sourceId}-${timestamp}-${uniqueSuffix}.log`);
76
+ }
77
+
78
+ /**
79
+ * Move the active log aside and release this isolate's descriptor, with nothing awaited in between:
80
+ * a rotation that yields to the event loop lets the logging loop that triggered it keep appending,
81
+ * which is the rate-dependent overshoot this whole change exists to remove.
82
+ */
83
+ export function rotateLogFileSync(logPath: string, rotatedLogDir: string, closeLogFile: () => void, activeStats?: any) {
84
+ const active = activeStats ?? statSync(logPath);
85
+ const archivePath = archivePathFor(logPath, rotatedLogDir);
86
+ renameSync(logPath, archivePath);
87
+ closeLogFile();
88
+ // From the archive, not from the stat above: another isolate can rotate this generation away and
89
+ // its sink recreate the pathname between the two, and then this rename moves an inode the earlier
90
+ // stat never saw. Announcing that stat's identity would name a generation nobody holds, so every
91
+ // peer would answer "released" while still appending to the one about to be destroyed.
92
+ let { ino, dev } = active;
93
+ try {
94
+ ({ ino, dev } = statSync(archivePath));
95
+ } catch {
96
+ // Already claimed by retention or another pass; the pre-rename identity is the best available.
97
+ }
98
+ return { logPath, archivePath, ino, dev, generation: nextGenerationId() };
99
+ }
100
+
101
+ /**
102
+ * Publish an archived generation: ask every enumerable in-process peer to release it, then compress
103
+ * it if requested. The plain archive is only unlinked once that release is proven.
104
+ */
105
+ export async function publishArchivedGeneration(
106
+ generation: any,
107
+ compress?: boolean,
108
+ reportCompressionError?: (error: any) => void
109
+ ) {
110
+ if (!(await requestGenerationClose(generation))) {
111
+ rememberUnprovenArchive(generation.archivePath, { generation, compress });
112
+ return generation.archivePath;
113
+ }
114
+ // A worker's peer set is eventually consistent while replacements join. Its release request can
115
+ // make peers reopen promptly, but only the main thread has an authoritative worker list and may
116
+ // destroy the plain archive; its audit will obtain a fresh proof for the archived snapshot.
117
+ if (compress && isMainThread) {
118
+ try {
119
+ return await compressArchive(generation.archivePath);
120
+ } catch (error) {
121
+ reportCompressionError?.(error);
122
+ }
123
+ }
124
+ return generation.archivePath;
125
+ }
126
+
127
+ // A compression retry queue, not the safety mechanism — safety is the release the tick proves before
128
+ // anything is destroyed. Bounded because rotation happens mostly in workers, which run no tick to
129
+ // drain it, and because the archives are found on disk anyway.
130
+ const MAX_UNPROVEN_ARCHIVES = 64;
131
+ const unprovenArchives = new Map<string, any>();
132
+
133
+ function rememberUnprovenArchive(archivePath: string, pending: any) {
134
+ unprovenArchives.set(archivePath, pending);
135
+ while (unprovenArchives.size > MAX_UNPROVEN_ARCHIVES) {
136
+ unprovenArchives.delete(unprovenArchives.keys().next().value);
137
+ }
138
+ }
139
+
140
+ export function isArchivePendingQuiescence(archivePath: string) {
141
+ return unprovenArchives.has(archivePath);
142
+ }
143
+
144
+ // Bounded per pass: a peer that never answers must not let the retry queue starve the audit work.
145
+ const MAX_RETRIES_PER_PASS = 4;
146
+
147
+ export async function retryPendingGenerations(options: any = {}) {
148
+ const { deadline = Infinity, shouldStop = () => false } = options;
149
+ let attempts = 0;
150
+ for (const [archivePath, pending] of [...unprovenArchives]) {
151
+ if (shouldStop() || Date.now() >= deadline) return;
152
+ if (attempts++ >= MAX_RETRIES_PER_PASS) return;
153
+ if (!existsSync(archivePath)) {
154
+ unprovenArchives.delete(archivePath);
155
+ continue;
156
+ }
157
+ if (!(await requestGenerationClose(pending.generation, deadline))) continue;
158
+ unprovenArchives.delete(archivePath);
159
+ // One archive that will not compress must not stop the rest of the queue.
160
+ if (pending.compress) await compressArchive(archivePath).catch(() => {});
161
+ }
162
+ }
163
+
164
+ /**
165
+ * Compress archives left plain by any isolate, from a listing taken before quiescence was proven.
166
+ * Write-path rotations happen mostly in the HTTP workers, whose unproven-archive bookkeeping the
167
+ * main thread cannot see, so without this a worker's archive would silently stay uncompressed for
168
+ * the life of the process however the operator configured `compress`.
169
+ * Only safe to call once quiescence has been proven for exactly this listing.
170
+ */
171
+ export async function compressPendingArchives(
172
+ rotatedLogDir: string,
173
+ files: string[],
174
+ liveLogPaths: Set<string>,
175
+ options: any = {}
176
+ ) {
177
+ const { deadline = Infinity, shouldStop = () => false } = options;
178
+ const present = new Set(files);
179
+ let firstError;
180
+ for (const file of files) {
181
+ if (shouldStop() || Date.now() >= deadline) break;
182
+ if (!file.endsWith('.log') || !isArchiveName(file)) continue;
183
+ const archivePath = join(rotatedLogDir, file);
184
+ if (liveLogPaths.has(resolve(archivePath)) || unprovenArchives.has(archivePath)) continue;
185
+ try {
186
+ if (present.has(`${file}.gz`)) {
187
+ // Both representations of one generation: a crash between the .gz rename and the source
188
+ // unlink leaves the plain copy behind, and every later pass skipped it as already done, so
189
+ // it survived until retention aged it out — or forever, retention being unset by default.
190
+ // The .gz is renamed into place whole, and this pass already proved every peer released it.
191
+ await fsProm.unlink(archivePath);
192
+ continue;
193
+ }
194
+ const result = await tryCompressArchive(archivePath);
195
+ if (!result.compressed) break;
196
+ } catch (error) {
197
+ if (error.code !== 'ENOENT' && !firstError) firstError = { error, file };
198
+ }
199
+ }
200
+ return firstError;
201
+ }
202
+
203
+ // One compression at a time per rotated directory. A small maxSize on a busy instance rotates
204
+ // hundreds of times a second, and a pipeline per rotation would exhaust descriptors and memory long
205
+ // before retention could run.
206
+ const compressionByDirectory = new Map<string, Promise<any>>();
207
+
208
+ export async function compressArchive(archivePath: string) {
209
+ return (await tryCompressArchive(archivePath)).path;
210
+ }
211
+
212
+ async function tryCompressArchive(archivePath: string) {
213
+ const directory = dirname(archivePath);
214
+ // Declined rather than queued: chaining would grow an unbounded list of pending jobs whenever
215
+ // rotation outruns gzip. The archive is left for the tick's bounded sweep, which is where an
216
+ // uncompressed rotation leaves it anyway. The slot is released by the promise this returns, not by
217
+ // a detached continuation, because that sweep awaits each call before making the next.
218
+ if (compressionByDirectory.has(directory)) return { path: archivePath, compressed: false };
219
+ const chain: Promise<any> = compressOneArchive(archivePath).finally(() => {
220
+ if (compressionByDirectory.get(directory) === chain) compressionByDirectory.delete(directory);
221
+ });
222
+ compressionByDirectory.set(directory, chain);
223
+ return { path: await chain, compressed: true };
224
+ }
225
+
226
+ async function compressOneArchive(archivePath: string) {
227
+ const compressedPath = `${archivePath}.gz`;
228
+ // Written to a temp file and renamed into place, so a crash mid-gzip can never leave a truncated
229
+ // `.gz` looking like the authoritative copy of a generation.
230
+ const temporaryPath = `${compressedPath}.${process.pid}-${threadId}-${rotationSequence++}.tmp`;
231
+ try {
232
+ await pipeline(createReadStream(archivePath), createGzip(), createWriteStream(temporaryPath));
233
+ await fsProm.rename(temporaryPath, compressedPath);
234
+ } catch (error) {
235
+ await fsProm.unlink(temporaryPath).catch(() => {});
236
+ throw error;
237
+ }
238
+ await fsProm.unlink(archivePath);
239
+ return compressedPath;
240
+ }
241
+
242
+ /**
243
+ * The write-path size guard. One subtraction and one branch per flush; one pathname stat once per
244
+ * quantum of this writer's own output.
245
+ */
246
+ export function createRotationGuard(options: any) {
247
+ const { logPath, maxBytes, rotatedLogDir, compress, getLogIdentity, closeLogFile, report, onRotated } = options;
248
+ const checkQuantum = Math.max(1, Math.floor(maxBytes / CHECK_QUANTUM_DIVISOR));
249
+ const logDir = dirname(logPath);
250
+ mkdirSync(rotatedLogDir, { recursive: true });
251
+ // The sink creates this one lazily, on the first append that gets ENOENT, and rotatedLogDir is not
252
+ // always under it — so without this the stat below throws ENOENT on a directory that is about to
253
+ // exist, and rotation stays off for the life of the process.
254
+ mkdirSync(logDir, { recursive: true });
255
+ // Rotation is a rename, and a rename across devices can never succeed. Refusing to build the guard
256
+ // here turns a misconfiguration that would otherwise fail closed on every write — ending file
257
+ // logging for the life of the process — into one startup error and today's unrotated behavior.
258
+ if (statSync(rotatedLogDir).dev !== statSync(logDir).dev) {
259
+ throw new Error(`the rotation directory ${rotatedLogDir} is on a different filesystem than ${logPath}`);
260
+ }
261
+ let bytesUntilCheck = checkQuantum;
262
+ let retryAfter = 0;
263
+ let rotationPending = false;
264
+ let rotating = false;
265
+ try {
266
+ // Seeded from the file as it actually is, so an instance restarted onto an already-oversized
267
+ // log rotates on its first write rather than on the first audit tick a minute later.
268
+ bytesUntilCheck = Math.min(checkQuantum, Math.max(0, maxBytes - statSync(logPath).size));
269
+ } catch {
270
+ // No log file yet.
271
+ }
272
+ return { beforeAppend, recordWrite, checkQuantum };
273
+
274
+ /**
275
+ * Try a pending rotation before the next append. A failed rotation leaves the active file
276
+ * writable: daemonized services discard stdio, so refusing the append would silently lose logs.
277
+ */
278
+ function beforeAppend() {
279
+ // `rotating` first: the rotation notice is written back through this same sink, and it must
280
+ // not re-enter a rotation that has not finished setting its own state.
281
+ if (rotating || !rotationPending) return true;
282
+ if (retryAfter > performance.now()) return true;
283
+ attemptRotation();
284
+ // If rotation is still unavailable, preserving records in the writable active log is safer
285
+ // than silently dropping them (scripted services have no stdio fallback).
286
+ return true;
287
+ }
288
+
289
+ function recordWrite(byteLength: number) {
290
+ bytesUntilCheck -= byteLength;
291
+ if (bytesUntilCheck > 0) return;
292
+ bytesUntilCheck = checkQuantum;
293
+ // `rotating` covers the rotation notice, which is written back through this same sink.
294
+ if (rotating || retryAfter > performance.now()) return;
295
+ attemptRotation();
296
+ }
297
+
298
+ function attemptRotation() {
299
+ rotating = true;
300
+ try {
301
+ checkAndRotate();
302
+ rotationPending = false;
303
+ } catch (error) {
304
+ // rename() reports ENOENT for a missing source AND for a missing destination directory, and
305
+ // only the first is benign. A missing source is the normal multi-writer outcome: another
306
+ // thread (or the audit tick) renamed the generation between this thread's stat and its
307
+ // rename, so the file is under the cap now, which is all this check wanted. A missing
308
+ // destination means no rotation can ever succeed, and treating that as a race would clear
309
+ // the cap check on every pass and let the log grow without a bound or a diagnostic. The
310
+ // destination is what is asked about: a peer can recreate the source pathname between the
311
+ // failed rename and this check, which would make the benign case look like the fatal one.
312
+ if (error.code === 'ENOENT' && existsSync(rotatedLogDir)) {
313
+ closeLogFile();
314
+ rotationPending = false;
315
+ return;
316
+ }
317
+ rotationPending = true;
318
+ retryAfter = performance.now() + ROTATION_RETRY_COOLDOWN;
319
+ closeLogFile();
320
+ report(`Harper cannot rotate its log file: ${error}`);
321
+ // A rotation target removed under a running instance is recoverable; recreate it here rather
322
+ // than on every rotation, so the common path keeps costing one stat and one rename.
323
+ if (error.code === 'ENOENT') {
324
+ try {
325
+ mkdirSync(rotatedLogDir, { recursive: true });
326
+ } catch {
327
+ // Reported above already; the retry will fail the same way and report again.
328
+ }
329
+ }
330
+ } finally {
331
+ rotating = false;
332
+ }
333
+ }
334
+
335
+ function checkAndRotate() {
336
+ let active;
337
+ try {
338
+ active = statSync(logPath);
339
+ } catch (error) {
340
+ if (error.code !== 'ENOENT') throw error;
341
+ // The generation this descriptor belongs to has already been rotated away by someone else.
342
+ closeLogFile();
343
+ return;
344
+ }
345
+ // The descriptor's identity is cached when it is opened, so the common checkpoint costs this
346
+ // one pathname stat rather than a stat plus an fstat.
347
+ if (!holdsGeneration(active)) {
348
+ closeLogFile();
349
+ return;
350
+ }
351
+ if (active.size < maxBytes) return;
352
+ const generation = rotateLogFileSync(logPath, rotatedLogDir, closeLogFile, active);
353
+ onRotated?.(generation.archivePath);
354
+ publishArchivedGeneration(generation, compress, (error) =>
355
+ report(`Harper could not compress a rotated log file: ${error}`)
356
+ ).catch((error) => report(`Harper could not publish a rotated log file: ${error}`));
357
+ }
358
+
359
+ function holdsGeneration(active: any) {
360
+ const identity = getLogIdentity();
361
+ if (!identity) return true;
362
+ // Some Windows filesystems report an unstable or zero ino, where identity cannot distinguish
363
+ // generations; there this defers to the size check rather than closing a descriptor at random.
364
+ if (identity.ino === 0 || active.ino === 0) return true;
365
+ return identity.ino === active.ino && identity.dev === active.dev;
366
+ }
367
+ }