@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/analytics/profile.ts +4 -0
- package/core/AGENTS.md +92 -6
- package/core/DESIGN.md +24 -0
- package/core/README.md +11 -10
- package/core/agent/agent.ts +203 -0
- package/core/agent/loop.ts +205 -0
- package/core/agent/operations.ts +148 -0
- package/core/agent/session.ts +187 -0
- package/core/agent/tools/fsTools.ts +276 -0
- package/core/agent/tools/httpFetchTool.ts +112 -0
- package/core/agent/tools/scheduleTool.ts +68 -0
- package/core/agent/toolset.ts +43 -0
- package/core/agent/types.ts +85 -0
- package/core/benchmarks/hnsw-search.js +157 -0
- package/core/benchmarks/ycsb/README.md +144 -0
- package/core/benchmarks/ycsb/analyze-profile.mts +120 -0
- package/core/benchmarks/ycsb/app/config.yaml +3 -0
- package/core/benchmarks/ycsb/app/schema.graphql +17 -0
- package/core/benchmarks/ycsb/harness.mts +328 -0
- package/core/benchmarks/ycsb/restClient.mts +120 -0
- package/core/benchmarks/ycsb/run-single-node.mts +97 -0
- package/core/benchmarks/ycsb/to-benchmark-json.mts +58 -0
- package/core/benchmarks/ycsb/workload.mts +499 -0
- package/core/benchmarks/ycsb/workload.test.mts +174 -0
- package/core/bin/{BinObjects.js → BinObjects.ts} +4 -5
- package/core/bin/cliCredentials.ts +133 -0
- package/core/bin/cliOperations.ts +339 -0
- package/core/bin/copyDb.ts +10 -10
- package/core/bin/deployRenderer.ts +196 -0
- package/core/bin/{harper.js → harper.ts} +48 -24
- package/core/bin/{install.js → install.ts} +3 -3
- package/core/bin/lite.ts +2 -0
- package/core/bin/login.ts +168 -0
- package/core/bin/logout.ts +11 -0
- package/core/bin/mcp/client.ts +407 -0
- package/core/bin/mcp/doctor.ts +189 -0
- package/core/bin/mcp/index.ts +80 -0
- package/core/bin/mcp/options.ts +122 -0
- package/core/bin/mcp/printConfig.ts +89 -0
- package/core/bin/multipartBuilder.ts +74 -0
- package/core/bin/{restart.js → restart.ts} +31 -32
- package/core/bin/{run.js → run.ts} +57 -46
- package/core/bin/sseConsumer.ts +126 -0
- package/core/bin/{status.js → status.ts} +10 -10
- package/core/bin/stop.ts +21 -0
- package/core/bin/upgrade.js +6 -6
- package/core/components/Application.ts +144 -18
- package/core/components/ApplicationScope.ts +2 -2
- package/core/components/ComponentV1.ts +2 -2
- package/core/components/EntryHandler.ts +159 -9
- package/core/components/OptionsWatcher.ts +75 -11
- package/core/components/Scope.ts +125 -15
- package/core/components/anthropic/index.ts +547 -0
- package/core/components/bedrock/index.ts +823 -0
- package/core/components/componentLoader.ts +63 -32
- package/core/components/deployLifecycle.ts +161 -0
- package/core/components/deploymentOperations.ts +173 -0
- package/core/components/deploymentRecorder.ts +402 -0
- package/core/components/deriveURLPath.ts +4 -4
- package/core/components/mcp/adapters/fastify.ts +87 -0
- package/core/components/mcp/adapters/harperHttp.ts +103 -0
- package/core/components/mcp/audit.ts +75 -0
- package/core/components/mcp/index.ts +134 -0
- package/core/components/mcp/jsonrpc.ts +134 -0
- package/core/components/mcp/lifecycle.ts +105 -0
- package/core/components/mcp/listChanged.ts +270 -0
- package/core/components/mcp/rateLimit.ts +217 -0
- package/core/components/mcp/resources.ts +593 -0
- package/core/components/mcp/session.ts +151 -0
- package/core/components/mcp/sessionRegistry.ts +140 -0
- package/core/components/mcp/toolRegistry.ts +292 -0
- package/core/components/mcp/tools/application.ts +603 -0
- package/core/components/mcp/tools/operations.ts +283 -0
- package/core/components/mcp/tools/schemas/derive.ts +256 -0
- package/core/components/mcp/tools/schemas/operations.ts +245 -0
- package/core/components/mcp/transport.ts +517 -0
- package/core/components/ollama/index.ts +316 -0
- package/core/components/openai/index.ts +563 -0
- package/core/components/operations.js +211 -60
- package/core/components/operationsValidation.js +3 -3
- package/core/components/packageComponent.ts +97 -29
- package/core/components/requestRestart.ts +17 -2
- package/core/components/status/crossThread.ts +14 -5
- package/core/components/status/errors.ts +1 -1
- package/core/config/RootConfigWatcher.ts +56 -2
- package/core/config/configUtils.js +29 -8
- package/core/config/harperConfigEnvVars.ts +1 -1
- package/core/dataLayer/{CreateAttributeObject.js → CreateAttributeObject.ts} +4 -3
- package/core/dataLayer/{CreateTableObject.js → CreateTableObject.ts} +2 -1
- package/core/dataLayer/{DataLayerObjects.js → DataLayerObjects.ts} +17 -9
- package/core/dataLayer/{DeleteBeforeObject.js → DeleteBeforeObject.ts} +2 -1
- package/core/dataLayer/{DeleteObject.js → DeleteObject.ts} +3 -2
- package/core/dataLayer/{DropAttributeObject.js → DropAttributeObject.ts} +2 -1
- package/core/dataLayer/{GetBackupObject.js → GetBackupObject.ts} +3 -2
- package/core/dataLayer/{InsertObject.js → InsertObject.ts} +3 -2
- package/core/dataLayer/{ReadAuditLogObject.js → ReadAuditLogObject.ts} +3 -2
- package/core/dataLayer/{SQLSearch.js → SQLSearch.ts} +97 -43
- package/core/dataLayer/{SearchByConditionsObject.js → SearchByConditionsObject.ts} +5 -6
- package/core/dataLayer/{SearchByHashObject.js → SearchByHashObject.ts} +2 -1
- package/core/dataLayer/{SearchObject.js → SearchObject.ts} +2 -1
- package/core/dataLayer/{SqlSearchObject.js → SqlSearchObject.ts} +2 -1
- package/core/dataLayer/{UpdateObject.js → UpdateObject.ts} +3 -2
- package/core/dataLayer/{UpsertObject.js → UpsertObject.ts} +3 -2
- package/core/dataLayer/{bulkLoad.js → bulkLoad.ts} +40 -49
- package/core/dataLayer/{delete.js → delete.ts} +21 -26
- package/core/dataLayer/{export.js → export.ts} +22 -26
- package/core/dataLayer/{getBackup.js → getBackup.ts} +7 -9
- package/core/dataLayer/harperBridge/BridgeMethods.ts +102 -0
- package/core/dataLayer/harperBridge/ResourceBridge.ts +27 -26
- package/core/dataLayer/harperBridge/TableSizeObject.ts +1 -0
- package/core/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +4 -4
- package/core/dataLayer/harperBridge/{harperBridge.js → harperBridge.ts} +3 -3
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +8 -6
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +4 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +6 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +5 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +4 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +6 -5
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +5 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +5 -5
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +3 -3
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +3 -2
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +5 -5
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +10 -8
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +4 -3
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +3 -3
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +6 -5
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +2 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +3 -2
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +2 -2
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +5 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +6 -3
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +4 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +5 -5
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +4 -4
- package/core/dataLayer/{hdbInfoController.js → hdbInfoController.ts} +20 -26
- package/core/dataLayer/{insert.js → insert.ts} +24 -24
- package/core/dataLayer/{readAuditLog.js → readAuditLog.ts} +8 -10
- package/core/dataLayer/{schema.js → schema.ts} +32 -44
- package/core/dataLayer/{schemaDescribe.js → schemaDescribe.ts} +23 -26
- package/core/dataLayer/{search.js → search.ts} +9 -15
- package/core/dataLayer/{transaction.js → transaction.ts} +2 -5
- package/core/dataLayer/{update.js → update.ts} +19 -22
- package/core/index.ts +5 -0
- package/core/json/systemSchema.json +65 -0
- package/core/launchServiceScripts/utility/checkNodeVersion.js +2 -0
- package/core/package-lock.json +9051 -4318
- package/core/resources/DESIGN.md +98 -0
- package/core/resources/DatabaseTransaction.ts +58 -30
- package/core/resources/ErrorResource.ts +2 -1
- package/core/resources/LMDBTransaction.ts +38 -29
- package/core/resources/RecordEncoder.ts +12 -4
- package/core/resources/RequestTarget.ts +2 -0
- package/core/resources/Resource.ts +89 -56
- package/core/resources/ResourceInterface.ts +44 -21
- package/core/resources/Resources.ts +8 -8
- package/core/resources/RocksIndexStore.ts +3 -0
- package/core/resources/RocksTransactionLogStore.ts +47 -28
- package/core/resources/Table.ts +386 -214
- package/core/resources/analytics/metadata.ts +1 -0
- package/core/resources/analytics/read.ts +24 -6
- package/core/resources/analytics/write.ts +240 -17
- package/core/resources/auditStore.ts +28 -19
- package/core/resources/blob.ts +53 -53
- package/core/resources/dataLoader.ts +4 -4
- package/core/resources/databases.ts +190 -71
- package/core/resources/graphql.ts +227 -164
- package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts +294 -65
- package/core/resources/indexes/vector.ts +17 -0
- package/core/resources/loadEnv.ts +21 -17
- package/core/resources/login.ts +5 -3
- package/core/resources/models/Models.ts +304 -0
- package/core/resources/models/TestBackend.ts +83 -0
- package/core/resources/models/agentLoop.ts +895 -0
- package/core/resources/models/analyticsTable.ts +199 -0
- package/core/resources/models/backendHelpers.ts +116 -0
- package/core/resources/models/backendRegistry.ts +66 -0
- package/core/resources/models/bootstrap.ts +135 -0
- package/core/resources/models/embedHook.ts +138 -0
- package/core/resources/models/types.ts +296 -0
- package/core/resources/replayLogs.ts +15 -7
- package/core/resources/roles.ts +62 -67
- package/core/resources/search.ts +355 -135
- package/core/resources/tracked.ts +8 -8
- package/core/resources/transaction.ts +8 -8
- package/core/resources/transactionBroadcast.ts +3 -3
- package/core/security/auth.ts +35 -26
- package/core/security/certificateVerification/crlVerification.ts +11 -4
- package/core/security/{cryptoHash.js → cryptoHash.ts} +3 -8
- package/core/security/data_objects/{PermissionAttributeResponseObject.js → PermissionAttributeResponseObject.ts} +4 -4
- package/core/security/data_objects/{PermissionResponseObject.js → PermissionResponseObject.ts} +12 -11
- package/core/security/data_objects/{PermissionTableResponseObject.js → PermissionTableResponseObject.ts} +6 -4
- package/core/security/{fastifyAuth.js → fastifyAuth.ts} +93 -20
- package/core/security/impersonation.ts +3 -3
- package/core/security/jsLoader.ts +22 -8
- package/core/security/{keys.js → keys.ts} +113 -121
- package/core/security/permissionsTranslator.js +2 -2
- package/core/security/{role.js → role.ts} +26 -33
- package/core/security/tokenAuthentication.ts +34 -7
- package/core/security/user.ts +26 -22
- package/core/server/DESIGN.md +139 -0
- package/core/server/DurableSubscriptionsSession.ts +67 -50
- package/core/server/REST.ts +120 -107
- package/core/server/Server.ts +31 -12
- package/core/server/fastifyRoutes/helpers/getCORSOptions.js +1 -1
- package/core/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +1 -1
- package/core/server/fastifyRoutes/helpers/getServerOptions.js +1 -1
- package/core/server/fastifyRoutes.ts +37 -33
- package/core/server/graphqlQuerying.ts +6 -5
- package/core/server/http.ts +517 -26
- package/core/server/itc/serverHandlers.js +75 -14
- package/core/server/jobs/{JobObject.js → JobObject.ts} +13 -6
- package/core/server/jobs/{jobProcess.js → jobProcess.ts} +20 -16
- package/core/server/jobs/{jobRunner.js → jobRunner.ts} +20 -21
- package/core/server/jobs/{jobs.js → jobs.ts} +41 -44
- package/core/server/loadRootComponents.js +1 -1
- package/core/server/middlewareChain.ts +270 -0
- package/core/server/mqtt.ts +35 -26
- package/core/server/nodeName.ts +2 -1
- package/core/server/operationsServer.ts +85 -10
- package/core/server/serverHelpers/Headers.ts +10 -8
- package/core/server/serverHelpers/JSONStream.ts +15 -5
- package/core/server/serverHelpers/Request.ts +370 -13
- package/core/server/serverHelpers/contentTypes.ts +42 -11
- package/core/server/serverHelpers/multipartParser.ts +152 -0
- package/core/server/serverHelpers/progressEmitter.ts +110 -0
- package/core/server/serverHelpers/serverHandlers.js +43 -7
- package/core/server/serverHelpers/serverUtilities.ts +40 -30
- package/core/server/static.ts +9 -6
- package/core/server/status/index.ts +2 -2
- package/core/server/storageReclamation.ts +39 -2
- package/core/server/threads/itc.js +7 -4
- package/core/server/threads/manageThreads.js +100 -26
- package/core/server/threads/socketRouter.ts +12 -275
- package/core/server/threads/threadServer.js +345 -105
- package/core/server/threads/workerProcessGuard.ts +93 -0
- package/core/server/throttle.ts +18 -0
- package/core/sqlTranslator/{SelectValidator.js → SelectValidator.ts} +41 -49
- package/core/sqlTranslator/{alasqlFunctionImporter.js → alasqlFunctionImporter.ts} +5 -5
- package/core/sqlTranslator/{deleteTranslator.js → deleteTranslator.ts} +13 -18
- package/core/sqlTranslator/{index.js → index.ts} +30 -33
- package/core/sqlTranslator/{sql_statement_bucket.js → sql_statement_bucket.ts} +49 -20
- package/core/static/README.md +10 -9
- package/core/system/000004.log +0 -0
- package/core/system/CURRENT +1 -0
- package/core/system/IDENTITY +1 -0
- package/core/system/LOCK +0 -0
- package/core/system/LOG +1351 -0
- package/core/system/MANIFEST-000005 +0 -0
- package/core/system/OPTIONS-000013 +607 -0
- package/core/system/OPTIONS-000015 +734 -0
- package/core/upgrade/{UpgradeObjects.js → UpgradeObjects.ts} +3 -6
- package/core/upgrade/directives/5-2-0.ts +49 -0
- package/core/upgrade/directives/{directivesController.js → directivesController.ts} +16 -16
- package/core/upgrade/{directivesManager.js → directivesManager.ts} +7 -11
- package/core/upgrade/{upgradePrompt.js → upgradePrompt.ts} +8 -14
- package/core/upgrade/{upgradeUtilities.js → upgradeUtilities.ts} +3 -7
- package/core/utility/{OperationFunctionCaller.js → OperationFunctionCaller.ts} +7 -7
- package/core/utility/{assignCmdEnvVariables.js → assignCmdEnvVariables.ts} +6 -8
- package/core/utility/{common_utils.js → common_utils.ts} +113 -139
- package/core/utility/environment/{environmentManager.js → environmentManager.ts} +34 -33
- package/core/utility/environment/systemInformation.ts +18 -4
- package/core/utility/errors/{commonErrors.js → commonErrors.ts} +9 -9
- package/core/utility/errors/{hdbError.js → hdbError.ts} +39 -45
- package/core/utility/expandEnvVar.ts +110 -0
- package/core/utility/functions/geo.js +2 -2
- package/core/utility/functions/sql/alaSQLExtension.js +1 -1
- package/core/utility/globalSchema.ts +30 -0
- package/core/utility/hdbTerms.ts +56 -0
- package/core/utility/install/checkJWTTokensExist.js +1 -1
- package/core/utility/install/{installer.js → installer.ts} +58 -59
- package/core/utility/installation.ts +2 -2
- package/core/utility/lmdb/{DBIDefinition.js → DBIDefinition.ts} +4 -1
- package/core/utility/lmdb/{DeleteRecordsResponseObject.js → DeleteRecordsResponseObject.ts} +2 -1
- package/core/utility/lmdb/{InsertRecordsResponseObject.js → InsertRecordsResponseObject.ts} +2 -1
- package/core/utility/lmdb/OpenDBIObject.ts +43 -0
- package/core/utility/lmdb/{OpenEnvironmentObject.js → OpenEnvironmentObject.ts} +19 -6
- package/core/utility/lmdb/{UpdateRecordsResponseObject.js → UpdateRecordsResponseObject.ts} +2 -1
- package/core/utility/lmdb/{UpsertRecordsResponseObject.js → UpsertRecordsResponseObject.ts} +2 -1
- package/core/utility/lmdb/{cleanLMDBMap.js → cleanLMDBMap.ts} +5 -5
- package/core/utility/lmdb/{commonUtility.js → commonUtility.ts} +13 -21
- package/core/utility/lmdb/{deleteUtility.js → deleteUtility.ts} +8 -12
- package/core/utility/lmdb/{environmentUtility.js → environmentUtility.ts} +43 -52
- package/core/utility/lmdb/{searchCursorFunctions.js → searchCursorFunctions.ts} +12 -26
- package/core/utility/lmdb/{searchUtility.js → searchUtility.ts} +75 -64
- package/core/utility/lmdb/{terms.js → terms.ts} +10 -23
- package/core/utility/lmdb/{writeUtility.js → writeUtility.ts} +37 -22
- package/core/utility/logging/{harper_logger.js → harper_logger.ts} +136 -89
- package/core/utility/logging/{logRotator.js → logRotator.ts} +13 -13
- package/core/utility/logging/logger.ts +1 -1
- package/core/utility/logging/{readLog.js → readLog.ts} +19 -19
- package/core/utility/logging/{transactionLog.js → transactionLog.ts} +10 -14
- package/core/utility/{mount_hdb.js → mount_hdb.ts} +15 -16
- package/core/utility/{npmUtilities.js → npmUtilities.ts} +14 -17
- package/core/utility/{operation_authorization.js → operation_authorization.ts} +173 -124
- package/core/utility/packageUtils.js +7 -16
- package/core/utility/password.ts +1 -1
- package/core/utility/processManagement/processManagement.js +2 -2
- package/core/utility/processManagement/servicesConfig.js +1 -1
- package/core/utility/{signalling.js → signalling.ts} +6 -11
- package/core/utility/watcherFallback.ts +74 -0
- package/core/validation/analyticsValidator.ts +43 -0
- package/core/validation/{bulkDeleteValidator.js → bulkDeleteValidator.ts} +5 -5
- package/core/validation/{check_permissions.js → check_permissions.ts} +3 -3
- package/core/validation/{common_validators.js → common_validators.ts} +12 -24
- package/core/validation/{configValidator.js → configValidator.ts} +114 -18
- package/core/validation/{deleteValidator.js → deleteValidator.ts} +5 -5
- package/core/validation/{fileLoadValidator.js → fileLoadValidator.ts} +12 -19
- package/core/validation/{insertValidator.js → insertValidator.ts} +5 -5
- package/core/validation/{installValidator.js → installValidator.ts} +8 -8
- package/core/validation/{readLogValidator.js → readLogValidator.ts} +10 -10
- package/core/validation/{role_validation.js → role_validation.ts} +26 -32
- package/core/validation/{schemaMetadataValidator.js → schemaMetadataValidator.ts} +5 -11
- package/core/validation/{searchValidator.js → searchValidator.ts} +12 -11
- package/core/validation/statusValidator.ts +1 -1
- package/core/validation/{transactionLogValidator.js → transactionLogValidator.ts} +4 -9
- package/core/validation/{user_validation.js → user_validation.ts} +4 -10
- package/core/validation/{validationWrapper.js → validationWrapper.ts} +3 -9
- package/dist/analytics/profile.js +4 -0
- package/dist/analytics/profile.js.map +1 -1
- package/dist/cloneNode/cloneNode.js +224 -12
- package/dist/cloneNode/cloneNode.js.map +1 -1
- package/dist/core/agent/agent.js +175 -0
- package/dist/core/agent/agent.js.map +1 -0
- package/dist/core/agent/loop.js +176 -0
- package/dist/core/agent/loop.js.map +1 -0
- package/dist/core/agent/operations.js +137 -0
- package/dist/core/agent/operations.js.map +1 -0
- package/dist/core/agent/session.js +182 -0
- package/dist/core/agent/session.js.map +1 -0
- package/dist/core/agent/tools/fsTools.js +286 -0
- package/dist/core/agent/tools/fsTools.js.map +1 -0
- package/dist/core/agent/tools/httpFetchTool.js +116 -0
- package/dist/core/agent/tools/httpFetchTool.js.map +1 -0
- package/dist/core/agent/tools/scheduleTool.js +54 -0
- package/dist/core/agent/tools/scheduleTool.js.map +1 -0
- package/dist/core/agent/toolset.js +33 -0
- package/dist/core/agent/toolset.js.map +1 -0
- package/dist/core/agent/types.js +10 -0
- package/dist/core/agent/types.js.map +1 -0
- package/dist/core/bin/BinObjects.js +6 -3
- package/dist/core/bin/BinObjects.js.map +1 -1
- package/dist/core/bin/cliCredentials.js +130 -0
- package/dist/core/bin/cliCredentials.js.map +1 -0
- package/dist/core/bin/cliOperations.js +254 -40
- package/dist/core/bin/cliOperations.js.map +1 -1
- package/dist/core/bin/copyDb.js +16 -16
- package/dist/core/bin/copyDb.js.map +1 -1
- package/dist/core/bin/deployRenderer.js +185 -0
- package/dist/core/bin/deployRenderer.js.map +1 -0
- package/dist/core/bin/harper.js +92 -31
- package/dist/core/bin/harper.js.map +1 -1
- package/dist/core/bin/install.js +41 -4
- package/dist/core/bin/install.js.map +1 -1
- package/dist/core/bin/lite.js +3 -4
- package/dist/core/bin/lite.js.map +1 -1
- package/dist/core/bin/login.js +158 -0
- package/dist/core/bin/login.js.map +1 -0
- package/dist/core/bin/logout.js +16 -0
- package/dist/core/bin/logout.js.map +1 -0
- package/dist/core/bin/mcp/client.js +395 -0
- package/dist/core/bin/mcp/client.js.map +1 -0
- package/dist/core/bin/mcp/doctor.js +193 -0
- package/dist/core/bin/mcp/doctor.js.map +1 -0
- package/dist/core/bin/mcp/index.js +81 -0
- package/dist/core/bin/mcp/index.js.map +1 -0
- package/dist/core/bin/mcp/options.js +113 -0
- package/dist/core/bin/mcp/options.js.map +1 -0
- package/dist/core/bin/mcp/printConfig.js +85 -0
- package/dist/core/bin/mcp/printConfig.js.map +1 -0
- package/dist/core/bin/multipartBuilder.js +55 -0
- package/dist/core/bin/multipartBuilder.js.map +1 -0
- package/dist/core/bin/restart.js +85 -48
- package/dist/core/bin/restart.js.map +1 -1
- package/dist/core/bin/run.js +123 -77
- package/dist/core/bin/run.js.map +1 -1
- package/dist/core/bin/sseConsumer.js +127 -0
- package/dist/core/bin/sseConsumer.js.map +1 -0
- package/dist/core/bin/status.js +48 -11
- package/dist/core/bin/status.js.map +1 -1
- package/dist/core/bin/stop.js +44 -7
- package/dist/core/bin/stop.js.map +1 -1
- package/dist/core/bin/upgrade.js +6 -6
- package/dist/core/components/Application.js +134 -28
- package/dist/core/components/Application.js.map +1 -1
- package/dist/core/components/ApplicationScope.js +2 -2
- package/dist/core/components/ComponentV1.js +5 -5
- package/dist/core/components/ComponentV1.js.map +1 -1
- package/dist/core/components/EntryHandler.js +153 -13
- package/dist/core/components/EntryHandler.js.map +1 -1
- package/dist/core/components/OptionsWatcher.js +72 -10
- package/dist/core/components/OptionsWatcher.js.map +1 -1
- package/dist/core/components/Scope.js +105 -9
- package/dist/core/components/Scope.js.map +1 -1
- package/dist/core/components/anthropic/index.js +428 -0
- package/dist/core/components/anthropic/index.js.map +1 -0
- package/dist/core/components/bedrock/index.js +734 -0
- package/dist/core/components/bedrock/index.js.map +1 -0
- package/dist/core/components/componentLoader.js +63 -38
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/deployLifecycle.js +156 -0
- package/dist/core/components/deployLifecycle.js.map +1 -0
- package/dist/core/components/deploymentOperations.js +185 -0
- package/dist/core/components/deploymentOperations.js.map +1 -0
- package/dist/core/components/deploymentRecorder.js +401 -0
- package/dist/core/components/deploymentRecorder.js.map +1 -0
- package/dist/core/components/deriveURLPath.js +2 -2
- package/dist/core/components/deriveURLPath.js.map +1 -1
- package/dist/core/components/mcp/adapters/fastify.js +66 -0
- package/dist/core/components/mcp/adapters/fastify.js.map +1 -0
- package/dist/core/components/mcp/adapters/harperHttp.js +78 -0
- package/dist/core/components/mcp/adapters/harperHttp.js.map +1 -0
- package/dist/core/components/mcp/audit.js +73 -0
- package/dist/core/components/mcp/audit.js.map +1 -0
- package/dist/core/components/mcp/index.js +109 -0
- package/dist/core/components/mcp/index.js.map +1 -0
- package/dist/core/components/mcp/jsonrpc.js +93 -0
- package/dist/core/components/mcp/jsonrpc.js.map +1 -0
- package/dist/core/components/mcp/lifecycle.js +79 -0
- package/dist/core/components/mcp/lifecycle.js.map +1 -0
- package/dist/core/components/mcp/listChanged.js +257 -0
- package/dist/core/components/mcp/listChanged.js.map +1 -0
- package/dist/core/components/mcp/rateLimit.js +226 -0
- package/dist/core/components/mcp/rateLimit.js.map +1 -0
- package/dist/core/components/mcp/resources.js +515 -0
- package/dist/core/components/mcp/resources.js.map +1 -0
- package/dist/core/components/mcp/session.js +170 -0
- package/dist/core/components/mcp/session.js.map +1 -0
- package/dist/core/components/mcp/sessionRegistry.js +124 -0
- package/dist/core/components/mcp/sessionRegistry.js.map +1 -0
- package/dist/core/components/mcp/toolRegistry.js +176 -0
- package/dist/core/components/mcp/toolRegistry.js.map +1 -0
- package/dist/core/components/mcp/tools/application.js +549 -0
- package/dist/core/components/mcp/tools/application.js.map +1 -0
- package/dist/core/components/mcp/tools/operations.js +303 -0
- package/dist/core/components/mcp/tools/operations.js.map +1 -0
- package/dist/core/components/mcp/tools/schemas/derive.js +216 -0
- package/dist/core/components/mcp/tools/schemas/derive.js.map +1 -0
- package/dist/core/components/mcp/tools/schemas/operations.js +243 -0
- package/dist/core/components/mcp/tools/schemas/operations.js.map +1 -0
- package/dist/core/components/mcp/transport.js +467 -0
- package/dist/core/components/mcp/transport.js.map +1 -0
- package/dist/core/components/ollama/index.js +239 -0
- package/dist/core/components/ollama/index.js.map +1 -0
- package/dist/core/components/openai/index.js +475 -0
- package/dist/core/components/openai/index.js.map +1 -0
- package/dist/core/components/operations.js +198 -52
- package/dist/core/components/operations.js.map +1 -1
- package/dist/core/components/operationsValidation.js +3 -3
- package/dist/core/components/packageComponent.js +87 -26
- package/dist/core/components/packageComponent.js.map +1 -1
- package/dist/core/components/requestRestart.js +12 -1
- package/dist/core/components/requestRestart.js.map +1 -1
- package/dist/core/components/status/crossThread.js +12 -5
- package/dist/core/components/status/crossThread.js.map +1 -1
- package/dist/core/components/status/errors.js +7 -7
- package/dist/core/config/RootConfigWatcher.js +52 -1
- package/dist/core/config/RootConfigWatcher.js.map +1 -1
- package/dist/core/config/configUtils.js +31 -8
- package/dist/core/config/configUtils.js.map +1 -1
- package/dist/core/config/harperConfigEnvVars.js +1 -1
- package/dist/core/config/harperConfigEnvVars.js.map +1 -1
- package/dist/core/dataLayer/CreateAttributeObject.js +4 -3
- package/dist/core/dataLayer/CreateAttributeObject.js.map +1 -1
- package/dist/core/dataLayer/CreateTableObject.js +2 -1
- package/dist/core/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/core/dataLayer/DataLayerObjects.js +19 -5
- package/dist/core/dataLayer/DataLayerObjects.js.map +1 -1
- package/dist/core/dataLayer/DeleteBeforeObject.js +2 -1
- package/dist/core/dataLayer/DeleteBeforeObject.js.map +1 -1
- package/dist/core/dataLayer/DeleteObject.js +4 -3
- package/dist/core/dataLayer/DeleteObject.js.map +1 -1
- package/dist/core/dataLayer/DropAttributeObject.js +2 -1
- package/dist/core/dataLayer/DropAttributeObject.js.map +1 -1
- package/dist/core/dataLayer/GetBackupObject.js +4 -3
- package/dist/core/dataLayer/GetBackupObject.js.map +1 -1
- package/dist/core/dataLayer/InsertObject.js +4 -3
- package/dist/core/dataLayer/InsertObject.js.map +1 -1
- package/dist/core/dataLayer/ReadAuditLogObject.js +4 -3
- package/dist/core/dataLayer/ReadAuditLogObject.js.map +1 -1
- package/dist/core/dataLayer/SQLSearch.js +140 -78
- package/dist/core/dataLayer/SQLSearch.js.map +1 -1
- package/dist/core/dataLayer/SearchByConditionsObject.js +5 -7
- package/dist/core/dataLayer/SearchByConditionsObject.js.map +1 -1
- package/dist/core/dataLayer/SearchByHashObject.js +2 -1
- package/dist/core/dataLayer/SearchByHashObject.js.map +1 -1
- package/dist/core/dataLayer/SearchObject.js +2 -1
- package/dist/core/dataLayer/SearchObject.js.map +1 -1
- package/dist/core/dataLayer/SqlSearchObject.js +2 -1
- package/dist/core/dataLayer/SqlSearchObject.js.map +1 -1
- package/dist/core/dataLayer/UpdateObject.js +4 -3
- package/dist/core/dataLayer/UpdateObject.js.map +1 -1
- package/dist/core/dataLayer/UpsertObject.js +4 -3
- package/dist/core/dataLayer/UpsertObject.js.map +1 -1
- package/dist/core/dataLayer/bulkLoad.js +122 -88
- package/dist/core/dataLayer/bulkLoad.js.map +1 -1
- package/dist/core/dataLayer/delete.js +74 -39
- package/dist/core/dataLayer/delete.js.map +1 -1
- package/dist/core/dataLayer/export.js +90 -55
- package/dist/core/dataLayer/export.js.map +1 -1
- package/dist/core/dataLayer/getBackup.js +43 -11
- package/dist/core/dataLayer/getBackup.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/BridgeMethods.js +41 -20
- package/dist/core/dataLayer/harperBridge/BridgeMethods.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/ResourceBridge.js +26 -25
- package/dist/core/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/TableSizeObject.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +4 -4
- package/dist/core/dataLayer/harperBridge/harperBridge.js +38 -4
- package/dist/core/dataLayer/harperBridge/harperBridge.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +7 -6
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +5 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +5 -5
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +5 -5
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +3 -3
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +2 -2
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +5 -5
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +8 -8
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +3 -3
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +3 -3
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +5 -5
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +2 -2
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +2 -2
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +5 -3
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +2 -2
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +4 -4
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +5 -5
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +4 -4
- package/dist/core/dataLayer/hdbInfoController.js +66 -31
- package/dist/core/dataLayer/hdbInfoController.js.map +1 -1
- package/dist/core/dataLayer/insert.js +63 -28
- package/dist/core/dataLayer/insert.js.map +1 -1
- package/dist/core/dataLayer/readAuditLog.js +45 -13
- package/dist/core/dataLayer/readAuditLog.js.map +1 -1
- package/dist/core/dataLayer/schema.js +124 -89
- package/dist/core/dataLayer/schema.js.map +1 -1
- package/dist/core/dataLayer/schemaDescribe.js +78 -41
- package/dist/core/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/core/dataLayer/search.js +12 -13
- package/dist/core/dataLayer/search.js.map +1 -1
- package/dist/core/dataLayer/transaction.js +3 -4
- package/dist/core/dataLayer/transaction.js.map +1 -1
- package/dist/core/dataLayer/update.js +53 -18
- package/dist/core/dataLayer/update.js.map +1 -1
- package/dist/core/globals.js +1 -0
- package/dist/core/globals.js.map +1 -1
- package/dist/core/index.js +4 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/json/systemSchema.json +65 -0
- package/dist/core/launchServiceScripts/utility/checkNodeVersion.js +3 -0
- package/dist/core/launchServiceScripts/utility/checkNodeVersion.js.map +1 -1
- package/dist/core/resources/DatabaseTransaction.js +39 -15
- package/dist/core/resources/DatabaseTransaction.js.map +1 -1
- package/dist/core/resources/ErrorResource.js +3 -1
- package/dist/core/resources/ErrorResource.js.map +1 -1
- package/dist/core/resources/LMDBTransaction.js +18 -7
- package/dist/core/resources/LMDBTransaction.js.map +1 -1
- package/dist/core/resources/RecordEncoder.js +5 -2
- package/dist/core/resources/RecordEncoder.js.map +1 -1
- package/dist/core/resources/RequestTarget.js.map +1 -1
- package/dist/core/resources/Resource.js +37 -10
- package/dist/core/resources/Resource.js.map +1 -1
- package/dist/core/resources/ResourceInterface.js +20 -0
- package/dist/core/resources/ResourceInterface.js.map +1 -1
- package/dist/core/resources/Resources.js +7 -6
- package/dist/core/resources/Resources.js.map +1 -1
- package/dist/core/resources/RocksIndexStore.js +3 -0
- package/dist/core/resources/RocksIndexStore.js.map +1 -1
- package/dist/core/resources/RocksTransactionLogStore.js +46 -27
- package/dist/core/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/core/resources/Table.js +267 -107
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/analytics/metadata.js +1 -0
- package/dist/core/resources/analytics/metadata.js.map +1 -1
- package/dist/core/resources/analytics/read.js +16 -5
- package/dist/core/resources/analytics/read.js.map +1 -1
- package/dist/core/resources/analytics/write.js +232 -20
- package/dist/core/resources/analytics/write.js.map +1 -1
- package/dist/core/resources/auditStore.js +13 -8
- package/dist/core/resources/auditStore.js.map +1 -1
- package/dist/core/resources/blob.js +8 -8
- package/dist/core/resources/blob.js.map +1 -1
- package/dist/core/resources/dataLoader.js +13 -13
- package/dist/core/resources/databases.js +160 -52
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/resources/graphql.js +224 -174
- package/dist/core/resources/graphql.js.map +1 -1
- package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js +308 -59
- package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/core/resources/indexes/vector.js +14 -0
- package/dist/core/resources/indexes/vector.js.map +1 -1
- package/dist/core/resources/loadEnv.js +21 -18
- package/dist/core/resources/loadEnv.js.map +1 -1
- package/dist/core/resources/login.js +5 -4
- package/dist/core/resources/login.js.map +1 -1
- package/dist/core/resources/models/Models.js +265 -0
- package/dist/core/resources/models/Models.js.map +1 -0
- package/dist/core/resources/models/TestBackend.js +71 -0
- package/dist/core/resources/models/TestBackend.js.map +1 -0
- package/dist/core/resources/models/agentLoop.js +746 -0
- package/dist/core/resources/models/agentLoop.js.map +1 -0
- package/dist/core/resources/models/analyticsTable.js +166 -0
- package/dist/core/resources/models/analyticsTable.js.map +1 -0
- package/dist/core/resources/models/backendHelpers.js +109 -0
- package/dist/core/resources/models/backendHelpers.js.map +1 -0
- package/dist/core/resources/models/backendRegistry.js +54 -0
- package/dist/core/resources/models/backendRegistry.js.map +1 -0
- package/dist/core/resources/models/bootstrap.js +112 -0
- package/dist/core/resources/models/bootstrap.js.map +1 -0
- package/dist/core/resources/models/embedHook.js +127 -0
- package/dist/core/resources/models/embedHook.js.map +1 -0
- package/dist/core/resources/models/types.js +11 -0
- package/dist/core/resources/models/types.js.map +1 -0
- package/dist/core/resources/replayLogs.js +7 -1
- package/dist/core/resources/replayLogs.js.map +1 -1
- package/dist/core/resources/roles.js +67 -71
- package/dist/core/resources/roles.js.map +1 -1
- package/dist/core/resources/search.js +349 -140
- package/dist/core/resources/search.js.map +1 -1
- package/dist/core/resources/tracked.js +14 -14
- package/dist/core/resources/tracked.js.map +1 -1
- package/dist/core/resources/transaction.js +1 -1
- package/dist/core/resources/transaction.js.map +1 -1
- package/dist/core/resources/transactionBroadcast.js.map +1 -1
- package/dist/core/security/auth.js +34 -25
- package/dist/core/security/auth.js.map +1 -1
- package/dist/core/security/certificateVerification/crlVerification.js +7 -1
- package/dist/core/security/certificateVerification/crlVerification.js.map +1 -1
- package/dist/core/security/cryptoHash.js +37 -5
- package/dist/core/security/cryptoHash.js.map +1 -1
- package/dist/core/security/data_objects/PermissionAttributeResponseObject.js +4 -1
- package/dist/core/security/data_objects/PermissionAttributeResponseObject.js.map +1 -1
- package/dist/core/security/data_objects/PermissionResponseObject.js +15 -8
- package/dist/core/security/data_objects/PermissionResponseObject.js.map +1 -1
- package/dist/core/security/data_objects/PermissionTableResponseObject.js +6 -1
- package/dist/core/security/data_objects/PermissionTableResponseObject.js.map +1 -1
- package/dist/core/security/fastifyAuth.js +131 -22
- package/dist/core/security/fastifyAuth.js.map +1 -1
- package/dist/core/security/impersonation.js +15 -15
- package/dist/core/security/jsLoader.js +18 -5
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/security/keys.js +160 -114
- package/dist/core/security/keys.js.map +1 -1
- package/dist/core/security/permissionsTranslator.js +2 -2
- package/dist/core/security/role.js +70 -35
- package/dist/core/security/role.js.map +1 -1
- package/dist/core/security/tokenAuthentication.js +57 -27
- package/dist/core/security/tokenAuthentication.js.map +1 -1
- package/dist/core/security/user.js +74 -38
- package/dist/core/security/user.js.map +1 -1
- package/dist/core/server/DurableSubscriptionsSession.js +71 -55
- package/dist/core/server/DurableSubscriptionsSession.js.map +1 -1
- package/dist/core/server/REST.js +17 -16
- package/dist/core/server/REST.js.map +1 -1
- package/dist/core/server/Server.js +1 -1
- package/dist/core/server/Server.js.map +1 -1
- package/dist/core/server/fastifyRoutes/helpers/getCORSOptions.js +1 -1
- package/dist/core/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +1 -1
- package/dist/core/server/fastifyRoutes/helpers/getServerOptions.js +1 -1
- package/dist/core/server/fastifyRoutes.js +34 -30
- package/dist/core/server/fastifyRoutes.js.map +1 -1
- package/dist/core/server/graphqlQuerying.js +8 -7
- package/dist/core/server/graphqlQuerying.js.map +1 -1
- package/dist/core/server/http.js +523 -47
- package/dist/core/server/http.js.map +1 -1
- package/dist/core/server/itc/serverHandlers.js +66 -15
- package/dist/core/server/itc/serverHandlers.js.map +1 -1
- package/dist/core/server/jobs/JobObject.js +53 -7
- package/dist/core/server/jobs/JobObject.js.map +1 -1
- package/dist/core/server/jobs/jobProcess.js +64 -24
- package/dist/core/server/jobs/jobProcess.js.map +1 -1
- package/dist/core/server/jobs/jobRunner.js +68 -30
- package/dist/core/server/jobs/jobRunner.js.map +1 -1
- package/dist/core/server/jobs/jobs.js +93 -61
- package/dist/core/server/jobs/jobs.js.map +1 -1
- package/dist/core/server/loadRootComponents.js +1 -1
- package/dist/core/server/middlewareChain.js +252 -0
- package/dist/core/server/middlewareChain.js.map +1 -0
- package/dist/core/server/mqtt.js +22 -17
- package/dist/core/server/mqtt.js.map +1 -1
- package/dist/core/server/nodeName.js +44 -13
- package/dist/core/server/nodeName.js.map +1 -1
- package/dist/core/server/operationsServer.js +106 -33
- package/dist/core/server/operationsServer.js.map +1 -1
- package/dist/core/server/serverHelpers/Headers.js +2 -0
- package/dist/core/server/serverHelpers/Headers.js.map +1 -1
- package/dist/core/server/serverHelpers/JSONStream.js +12 -3
- package/dist/core/server/serverHelpers/JSONStream.js.map +1 -1
- package/dist/core/server/serverHelpers/Request.js +370 -9
- package/dist/core/server/serverHelpers/Request.js.map +1 -1
- package/dist/core/server/serverHelpers/contentTypes.js +36 -7
- package/dist/core/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/core/server/serverHelpers/multipartParser.js +142 -0
- package/dist/core/server/serverHelpers/multipartParser.js.map +1 -0
- package/dist/core/server/serverHelpers/progressEmitter.js +103 -0
- package/dist/core/server/serverHelpers/progressEmitter.js.map +1 -0
- package/dist/core/server/serverHelpers/serverHandlers.js +38 -7
- package/dist/core/server/serverHelpers/serverHandlers.js.map +1 -1
- package/dist/core/server/serverHelpers/serverUtilities.js +97 -93
- package/dist/core/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/core/server/static.js +8 -5
- package/dist/core/server/static.js.map +1 -1
- package/dist/core/server/status/index.js +3 -3
- package/dist/core/server/storageReclamation.js +68 -9
- package/dist/core/server/storageReclamation.js.map +1 -1
- package/dist/core/server/threads/itc.js +7 -4
- package/dist/core/server/threads/itc.js.map +1 -1
- package/dist/core/server/threads/manageThreads.js +110 -26
- package/dist/core/server/threads/manageThreads.js.map +1 -1
- package/dist/core/server/threads/socketRouter.js +8 -271
- package/dist/core/server/threads/socketRouter.js.map +1 -1
- package/dist/core/server/threads/threadServer.js +360 -118
- package/dist/core/server/threads/threadServer.js.map +1 -1
- package/dist/core/server/threads/workerProcessGuard.js +114 -0
- package/dist/core/server/threads/workerProcessGuard.js.map +1 -0
- package/dist/core/server/throttle.js +17 -0
- package/dist/core/server/throttle.js.map +1 -1
- package/dist/core/sqlTranslator/SelectValidator.js +86 -47
- package/dist/core/sqlTranslator/SelectValidator.js.map +1 -1
- package/dist/core/sqlTranslator/alasqlFunctionImporter.js +40 -3
- package/dist/core/sqlTranslator/alasqlFunctionImporter.js.map +1 -1
- package/dist/core/sqlTranslator/deleteTranslator.js +48 -14
- package/dist/core/sqlTranslator/deleteTranslator.js.map +1 -1
- package/dist/core/sqlTranslator/index.js +69 -30
- package/dist/core/sqlTranslator/index.js.map +1 -1
- package/dist/core/sqlTranslator/sql_statement_bucket.js +55 -13
- package/dist/core/sqlTranslator/sql_statement_bucket.js.map +1 -1
- package/dist/core/upgrade/UpgradeObjects.js +37 -4
- package/dist/core/upgrade/UpgradeObjects.js.map +1 -1
- package/dist/core/upgrade/directives/5-2-0.js +77 -0
- package/dist/core/upgrade/directives/5-2-0.js.map +1 -0
- package/dist/core/upgrade/directives/directivesController.js +52 -11
- package/dist/core/upgrade/directives/directivesController.js.map +1 -1
- package/dist/core/upgrade/directivesManager.js +53 -18
- package/dist/core/upgrade/directivesManager.js.map +1 -1
- package/dist/core/upgrade/upgradePrompt.js +65 -30
- package/dist/core/upgrade/upgradePrompt.js.map +1 -1
- package/dist/core/upgrade/upgradeUtilities.js +37 -5
- package/dist/core/upgrade/upgradeUtilities.js.map +1 -1
- package/dist/core/utility/OperationFunctionCaller.js +45 -10
- package/dist/core/utility/OperationFunctionCaller.js.map +1 -1
- package/dist/core/utility/assignCmdEnvVariables.js +8 -4
- package/dist/core/utility/assignCmdEnvVariables.js.map +1 -1
- package/dist/core/utility/common_utils.js +140 -79
- package/dist/core/utility/common_utils.js.map +1 -1
- package/dist/core/utility/environment/environmentManager.js +75 -29
- package/dist/core/utility/environment/environmentManager.js.map +1 -1
- package/dist/core/utility/environment/systemInformation.js +27 -16
- package/dist/core/utility/environment/systemInformation.js.map +1 -1
- package/dist/core/utility/errors/commonErrors.js +49 -18
- package/dist/core/utility/errors/commonErrors.js.map +1 -1
- package/dist/core/utility/errors/hdbError.js +65 -26
- package/dist/core/utility/errors/hdbError.js.map +1 -1
- package/dist/core/utility/expandEnvVar.js +113 -0
- package/dist/core/utility/expandEnvVar.js.map +1 -0
- package/dist/core/utility/functions/geo.js +2 -2
- package/dist/core/utility/functions/sql/alaSQLExtension.js +1 -1
- package/dist/core/utility/globalSchema.js +14 -11
- package/dist/core/utility/globalSchema.js.map +1 -1
- package/dist/core/utility/hdbTerms.js +56 -0
- package/dist/core/utility/hdbTerms.js.map +1 -1
- package/dist/core/utility/install/checkJWTTokensExist.js +1 -1
- package/dist/core/utility/install/installer.js +106 -70
- package/dist/core/utility/install/installer.js.map +1 -1
- package/dist/core/utility/installation.js +3 -3
- package/dist/core/utility/lmdb/DBIDefinition.js +5 -1
- package/dist/core/utility/lmdb/DBIDefinition.js.map +1 -1
- package/dist/core/utility/lmdb/DeleteRecordsResponseObject.js +2 -1
- package/dist/core/utility/lmdb/DeleteRecordsResponseObject.js.map +1 -1
- package/dist/core/utility/lmdb/InsertRecordsResponseObject.js +2 -1
- package/dist/core/utility/lmdb/InsertRecordsResponseObject.js.map +1 -1
- package/dist/core/utility/lmdb/OpenDBIObject.js +54 -6
- package/dist/core/utility/lmdb/OpenDBIObject.js.map +1 -1
- package/dist/core/utility/lmdb/OpenEnvironmentObject.js +52 -4
- package/dist/core/utility/lmdb/OpenEnvironmentObject.js.map +1 -1
- package/dist/core/utility/lmdb/UpdateRecordsResponseObject.js +2 -1
- package/dist/core/utility/lmdb/UpdateRecordsResponseObject.js.map +1 -1
- package/dist/core/utility/lmdb/UpsertRecordsResponseObject.js +2 -1
- package/dist/core/utility/lmdb/UpsertRecordsResponseObject.js.map +1 -1
- package/dist/core/utility/lmdb/cleanLMDBMap.js +44 -7
- package/dist/core/utility/lmdb/cleanLMDBMap.js.map +1 -1
- package/dist/core/utility/lmdb/commonUtility.js +46 -17
- package/dist/core/utility/lmdb/commonUtility.js.map +1 -1
- package/dist/core/utility/lmdb/deleteUtility.js +51 -16
- package/dist/core/utility/lmdb/deleteUtility.js.map +1 -1
- package/dist/core/utility/lmdb/environmentUtility.js +91 -51
- package/dist/core/utility/lmdb/environmentUtility.js.map +1 -1
- package/dist/core/utility/lmdb/searchCursorFunctions.js +46 -14
- package/dist/core/utility/lmdb/searchCursorFunctions.js.map +1 -1
- package/dist/core/utility/lmdb/searchUtility.js +91 -55
- package/dist/core/utility/lmdb/searchUtility.js.map +1 -1
- package/dist/core/utility/lmdb/terms.js +12 -22
- package/dist/core/utility/lmdb/terms.js.map +1 -1
- package/dist/core/utility/lmdb/writeUtility.js +61 -28
- package/dist/core/utility/lmdb/writeUtility.js.map +1 -1
- package/dist/core/utility/logging/harper_logger.js +176 -81
- package/dist/core/utility/logging/harper_logger.js.map +1 -1
- package/dist/core/utility/logging/logRotator.js +65 -28
- package/dist/core/utility/logging/logRotator.js.map +1 -1
- package/dist/core/utility/logging/logger.js +4 -4
- package/dist/core/utility/logging/readLog.js +54 -17
- package/dist/core/utility/logging/readLog.js.map +1 -1
- package/dist/core/utility/logging/transactionLog.js +51 -16
- package/dist/core/utility/logging/transactionLog.js.map +1 -1
- package/dist/core/utility/mount_hdb.js +54 -17
- package/dist/core/utility/mount_hdb.js.map +1 -1
- package/dist/core/utility/npmUtilities.js +54 -19
- package/dist/core/utility/npmUtilities.js.map +1 -1
- package/dist/core/utility/operation_authorization.js +135 -86
- package/dist/core/utility/operation_authorization.js.map +1 -1
- package/dist/core/utility/packageUtils.js +7 -17
- package/dist/core/utility/packageUtils.js.map +1 -1
- package/dist/core/utility/password.js +2 -2
- package/dist/core/utility/processManagement/processManagement.js +2 -2
- package/dist/core/utility/processManagement/servicesConfig.js +1 -1
- package/dist/core/utility/signalling.js +51 -16
- package/dist/core/utility/signalling.js.map +1 -1
- package/dist/core/utility/watcherFallback.js +73 -0
- package/dist/core/utility/watcherFallback.js.map +1 -0
- package/dist/core/validation/analyticsValidator.js +79 -0
- package/dist/core/validation/analyticsValidator.js.map +1 -0
- package/dist/core/validation/bulkDeleteValidator.js +49 -11
- package/dist/core/validation/bulkDeleteValidator.js.map +1 -1
- package/dist/core/validation/check_permissions.js +38 -3
- package/dist/core/validation/check_permissions.js.map +1 -1
- package/dist/core/validation/common_validators.js +62 -31
- package/dist/core/validation/common_validators.js.map +1 -1
- package/dist/core/validation/configValidator.js +189 -54
- package/dist/core/validation/configValidator.js.map +1 -1
- package/dist/core/validation/deleteValidator.js +49 -11
- package/dist/core/validation/deleteValidator.js.map +1 -1
- package/dist/core/validation/fileLoadValidator.js +67 -32
- package/dist/core/validation/fileLoadValidator.js.map +1 -1
- package/dist/core/validation/insertValidator.js +48 -10
- package/dist/core/validation/insertValidator.js.map +1 -1
- package/dist/core/validation/installValidator.js +47 -10
- package/dist/core/validation/installValidator.js.map +1 -1
- package/dist/core/validation/readLogValidator.js +60 -22
- package/dist/core/validation/readLogValidator.js.map +1 -1
- package/dist/core/validation/role_validation.js +55 -19
- package/dist/core/validation/role_validation.js.map +1 -1
- package/dist/core/validation/schemaMetadataValidator.js +11 -12
- package/dist/core/validation/schemaMetadataValidator.js.map +1 -1
- package/dist/core/validation/searchValidator.js +82 -43
- package/dist/core/validation/searchValidator.js.map +1 -1
- package/dist/core/validation/transactionLogValidator.js +52 -17
- package/dist/core/validation/transactionLogValidator.js.map +1 -1
- package/dist/core/validation/user_validation.js +38 -6
- package/dist/core/validation/user_validation.js.map +1 -1
- package/dist/core/validation/validationWrapper.js +4 -5
- package/dist/core/validation/validationWrapper.js.map +1 -1
- package/dist/licensing/usageLicensing.js +30 -21
- package/dist/licensing/usageLicensing.js.map +1 -1
- package/dist/replication/knownNodes.js +173 -39
- package/dist/replication/knownNodes.js.map +1 -1
- package/dist/replication/replicationConnection.js +441 -85
- package/dist/replication/replicationConnection.js.map +1 -1
- package/dist/replication/replicator.js +44 -26
- package/dist/replication/replicator.js.map +1 -1
- package/dist/replication/setNode.js +22 -2
- package/dist/replication/setNode.js.map +1 -1
- package/dist/replication/subscriptionManager.js +121 -9
- package/dist/replication/subscriptionManager.js.map +1 -1
- package/dist/security/certificate.js +41 -6
- package/dist/security/certificate.js.map +1 -1
- package/dist/security/sshKeyOperations.js +35 -2
- package/dist/security/sshKeyOperations.js.map +1 -1
- package/licensing/usageLicensing.ts +32 -37
- package/npm-shrinkwrap.json +8994 -4305
- package/package.json +16 -11
- package/replication/DESIGN.md +139 -0
- package/replication/knownNodes.ts +166 -43
- package/replication/replicationConnection.ts +475 -92
- package/replication/replicator.ts +42 -25
- package/replication/setNode.ts +28 -9
- package/replication/subscriptionManager.ts +138 -14
- package/security/certificate.ts +8 -4
- package/security/sshKeyOperations.ts +1 -1
- package/static/defaultConfig.yaml +1 -0
- package/studio/web/assets/{index-CmtPP0YO.js → index-COfIkCT-.js} +5 -5
- package/studio/web/assets/index-COfIkCT-.js.map +1 -0
- package/studio/web/assets/{index.lazy-C8jvGtlu.js → index.lazy-CIvl7Fj9.js} +2 -2
- package/studio/web/assets/{index.lazy-C8jvGtlu.js.map → index.lazy-CIvl7Fj9.js.map} +1 -1
- package/studio/web/assets/{profile-O0DYlJUv.js → profile-B2ZVB--r.js} +2 -2
- package/studio/web/assets/{profile-O0DYlJUv.js.map → profile-B2ZVB--r.js.map} +1 -1
- package/studio/web/assets/{status-BIlJkJby.js → status-Db6WBmhf.js} +2 -2
- package/studio/web/assets/{status-BIlJkJby.js.map → status-Db6WBmhf.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/core/bin/cliOperations.js +0 -159
- package/core/bin/lite.js +0 -5
- package/core/bin/stop.js +0 -21
- package/core/dataLayer/harperBridge/BridgeMethods.js +0 -85
- package/core/utility/globalSchema.js +0 -35
- package/core/utility/lmdb/OpenDBIObject.js +0 -31
- package/studio/web/assets/index-CmtPP0YO.js.map +0 -1
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Summarizes V8 .cpuprofile files produced by `run-single-node.mts --profile`.
|
|
3
|
+
* Prints the hottest functions by self time and a per-module breakdown, which
|
|
4
|
+
* is the input for performance recommendations.
|
|
5
|
+
*
|
|
6
|
+
* node benchmarks/ycsb/analyze-profile.mts benchmarks/ycsb/results/profile [topN]
|
|
7
|
+
*/
|
|
8
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
|
|
11
|
+
interface CallFrame {
|
|
12
|
+
functionName: string;
|
|
13
|
+
url: string;
|
|
14
|
+
lineNumber: number;
|
|
15
|
+
}
|
|
16
|
+
interface ProfileNode {
|
|
17
|
+
id: number;
|
|
18
|
+
callFrame: CallFrame;
|
|
19
|
+
hitCount?: number;
|
|
20
|
+
children?: number[];
|
|
21
|
+
}
|
|
22
|
+
interface CpuProfile {
|
|
23
|
+
nodes: ProfileNode[];
|
|
24
|
+
samples: number[];
|
|
25
|
+
timeDeltas: number[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function moduleOf(url: string): string {
|
|
29
|
+
if (!url) return '(builtin/native)';
|
|
30
|
+
if (url.startsWith('node:')) return url;
|
|
31
|
+
const nm = url.lastIndexOf('node_modules/');
|
|
32
|
+
if (nm >= 0) {
|
|
33
|
+
const rest = url.slice(nm + 'node_modules/'.length).split('/');
|
|
34
|
+
return rest[0].startsWith('@') ? `${rest[0]}/${rest[1]}` : rest[0];
|
|
35
|
+
}
|
|
36
|
+
const dist = url.indexOf('/dist/');
|
|
37
|
+
if (dist >= 0) {
|
|
38
|
+
// dist/<area>/file.js — bucket by the top-level area (resources, server, ...)
|
|
39
|
+
return `harper/${url.slice(dist + '/dist/'.length).split('/')[0]}`;
|
|
40
|
+
}
|
|
41
|
+
return url.split('/').slice(-2).join('/');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// V8 synthetic frames — (program), (idle), (garbage collector) — carry their label in
|
|
45
|
+
// functionName with an empty url; bucket them on the name so they don't fold into native.
|
|
46
|
+
function moduleOfFrame(frame: CallFrame): string {
|
|
47
|
+
return frame.functionName.startsWith('(') ? frame.functionName : moduleOf(frame.url);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function fnKey(frame: CallFrame): string {
|
|
51
|
+
const mod = moduleOf(frame.url);
|
|
52
|
+
const name = frame.functionName || '(anonymous)';
|
|
53
|
+
return `${name} [${mod}]`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function analyzeFile(
|
|
57
|
+
path: string
|
|
58
|
+
): Promise<{ self: Map<string, number>; module: Map<string, number>; total: number }> {
|
|
59
|
+
const profile = JSON.parse(await readFile(path, 'utf8')) as CpuProfile;
|
|
60
|
+
const byId = new Map<number, ProfileNode>();
|
|
61
|
+
for (const node of profile.nodes) byId.set(node.id, node);
|
|
62
|
+
|
|
63
|
+
const selfMicros = new Map<number, number>();
|
|
64
|
+
for (let i = 0; i < profile.samples.length; i++) {
|
|
65
|
+
const id = profile.samples[i];
|
|
66
|
+
selfMicros.set(id, (selfMicros.get(id) ?? 0) + (profile.timeDeltas[i] ?? 0));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const self = new Map<string, number>();
|
|
70
|
+
const module = new Map<string, number>();
|
|
71
|
+
let total = 0;
|
|
72
|
+
for (const [id, micros] of selfMicros) {
|
|
73
|
+
const node = byId.get(id);
|
|
74
|
+
if (!node) continue;
|
|
75
|
+
const m = Math.max(0, micros);
|
|
76
|
+
total += m;
|
|
77
|
+
const key = fnKey(node.callFrame);
|
|
78
|
+
self.set(key, (self.get(key) ?? 0) + m);
|
|
79
|
+
const mod = moduleOfFrame(node.callFrame);
|
|
80
|
+
module.set(mod, (module.get(mod) ?? 0) + m);
|
|
81
|
+
}
|
|
82
|
+
return { self, module, total };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function topEntries(map: Map<string, number>, total: number, n: number): string[] {
|
|
86
|
+
return [...map.entries()]
|
|
87
|
+
.sort((a, b) => b[1] - a[1])
|
|
88
|
+
.slice(0, n)
|
|
89
|
+
.map(
|
|
90
|
+
([key, micros]) =>
|
|
91
|
+
` ${((micros / total) * 100).toFixed(1).padStart(5)}% ${(micros / 1000).toFixed(0).padStart(7)} ms ${key}`
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function main(): Promise<void> {
|
|
96
|
+
const dir = process.argv[2];
|
|
97
|
+
const topN = Number(process.argv[3] ?? 30);
|
|
98
|
+
if (!dir) throw new Error('usage: analyze-profile.mts <profile-dir> [topN]');
|
|
99
|
+
const files = (await readdir(dir)).filter((f) => f.endsWith('.cpuprofile'));
|
|
100
|
+
if (files.length === 0) throw new Error(`no .cpuprofile files in ${dir}`);
|
|
101
|
+
|
|
102
|
+
const self = new Map<string, number>();
|
|
103
|
+
const module = new Map<string, number>();
|
|
104
|
+
let total = 0;
|
|
105
|
+
for (const file of files) {
|
|
106
|
+
const r = await analyzeFile(join(dir, file));
|
|
107
|
+
for (const [k, v] of r.self) self.set(k, (self.get(k) ?? 0) + v);
|
|
108
|
+
for (const [k, v] of r.module) module.set(k, (module.get(k) ?? 0) + v);
|
|
109
|
+
total += r.total;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
process.stdout.write(`\nCPU profile summary — ${files.length} file(s), ${(total / 1000).toFixed(0)} ms of samples\n`);
|
|
113
|
+
process.stdout.write(`\nBy module / category (self time):\n${topEntries(module, total, 20).join('\n')}\n`);
|
|
114
|
+
process.stdout.write(`\nTop ${topN} functions (self time):\n${topEntries(self, total, topN).join('\n')}\n`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
main().catch((error) => {
|
|
118
|
+
console.error(error);
|
|
119
|
+
process.exit(1);
|
|
120
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# YCSB record table: a string primary key plus 10 unindexed string fields
|
|
2
|
+
# (the YCSB default record shape). Only the primary key is indexed, so reads,
|
|
3
|
+
# updates and key-range scans all go through the primary index — matching the
|
|
4
|
+
# YCSB access model.
|
|
5
|
+
type usertable @table @export {
|
|
6
|
+
id: ID @primaryKey
|
|
7
|
+
field0: String
|
|
8
|
+
field1: String
|
|
9
|
+
field2: String
|
|
10
|
+
field3: String
|
|
11
|
+
field4: String
|
|
12
|
+
field5: String
|
|
13
|
+
field6: String
|
|
14
|
+
field7: String
|
|
15
|
+
field8: String
|
|
16
|
+
field9: String
|
|
17
|
+
}
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared YCSB benchmark harness: CLI option parsing, the load + run-phase
|
|
3
|
+
* driver, results assembly, and console reporting. Both the single-node
|
|
4
|
+
* (harper) and 3-node cluster (harper-pro) entry points call into this so the
|
|
5
|
+
* workload definitions and result shape stay identical across them.
|
|
6
|
+
*/
|
|
7
|
+
import { parseArgs } from 'node:util';
|
|
8
|
+
import { execFileSync } from 'node:child_process';
|
|
9
|
+
import { writeFile, mkdir } from 'node:fs/promises';
|
|
10
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import {
|
|
13
|
+
WORKLOADS,
|
|
14
|
+
KeyState,
|
|
15
|
+
runOperations,
|
|
16
|
+
type DistributionName,
|
|
17
|
+
type LatencyStats,
|
|
18
|
+
type OperationType,
|
|
19
|
+
type PhaseResult,
|
|
20
|
+
} from './workload.mts';
|
|
21
|
+
import { createRestExecutor } from './restClient.mts';
|
|
22
|
+
|
|
23
|
+
export interface BenchmarkConfig {
|
|
24
|
+
records: number;
|
|
25
|
+
opsPerWorkload: number;
|
|
26
|
+
concurrency: number;
|
|
27
|
+
loadConcurrency: number;
|
|
28
|
+
fieldCount: number;
|
|
29
|
+
fieldLength: number;
|
|
30
|
+
maxScanLength: number;
|
|
31
|
+
warmupOps: number;
|
|
32
|
+
workloads: string[];
|
|
33
|
+
distribution?: DistributionName;
|
|
34
|
+
table: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface ParsedOptions {
|
|
38
|
+
config: BenchmarkConfig;
|
|
39
|
+
scale: string;
|
|
40
|
+
engine: string;
|
|
41
|
+
threads: number;
|
|
42
|
+
nodeCount: number;
|
|
43
|
+
out: string;
|
|
44
|
+
startupTimeoutMs: number;
|
|
45
|
+
settleMs: number;
|
|
46
|
+
profile: boolean;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const SCALE_PRESETS: Record<string, { records: number; opsPerWorkload: number; concurrency: number }> = {
|
|
50
|
+
quick: { records: 50_000, opsPerWorkload: 100_000, concurrency: 32 },
|
|
51
|
+
standard: { records: 200_000, opsPerWorkload: 500_000, concurrency: 64 },
|
|
52
|
+
heavy: { records: 1_000_000, opsPerWorkload: 4_000_000, concurrency: 128 },
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const DEFAULT_WORKLOAD_ORDER = ['C', 'B', 'A', 'F', 'D', 'E'];
|
|
56
|
+
|
|
57
|
+
export function parseOptions(argv: string[]): ParsedOptions {
|
|
58
|
+
const { values } = parseArgs({
|
|
59
|
+
args: argv,
|
|
60
|
+
options: {
|
|
61
|
+
'scale': { type: 'string', default: 'standard' },
|
|
62
|
+
'records': { type: 'string' },
|
|
63
|
+
'ops': { type: 'string' },
|
|
64
|
+
'concurrency': { type: 'string' },
|
|
65
|
+
'load-concurrency': { type: 'string' },
|
|
66
|
+
'fields': { type: 'string', default: '10' },
|
|
67
|
+
'field-length': { type: 'string', default: '100' },
|
|
68
|
+
'scan-max': { type: 'string', default: '100' },
|
|
69
|
+
'warmup': { type: 'string' },
|
|
70
|
+
'workloads': { type: 'string', default: DEFAULT_WORKLOAD_ORDER.join(',') },
|
|
71
|
+
'distribution': { type: 'string' },
|
|
72
|
+
'engine': { type: 'string', default: 'rocksdb' },
|
|
73
|
+
'threads': { type: 'string', default: '4' },
|
|
74
|
+
'nodes': { type: 'string', default: '3' },
|
|
75
|
+
'out': { type: 'string', default: join(import.meta.dirname, 'results') },
|
|
76
|
+
'startup-timeout': { type: 'string', default: '120000' },
|
|
77
|
+
'settle-ms': { type: 'string', default: '0' },
|
|
78
|
+
'profile': { type: 'boolean', default: false },
|
|
79
|
+
},
|
|
80
|
+
allowPositionals: false,
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const preset = SCALE_PRESETS[values.scale as string];
|
|
84
|
+
if (!preset) throw new Error(`unknown scale "${values.scale}" (expected ${Object.keys(SCALE_PRESETS).join(', ')})`);
|
|
85
|
+
|
|
86
|
+
const opsPerWorkload = values.ops ? Number(values.ops) : preset.opsPerWorkload;
|
|
87
|
+
const concurrency = values.concurrency ? Number(values.concurrency) : preset.concurrency;
|
|
88
|
+
const workloads = (values.workloads as string).split(',').map((w) => w.trim().toUpperCase());
|
|
89
|
+
for (const w of workloads) {
|
|
90
|
+
if (!WORKLOADS[w])
|
|
91
|
+
throw new Error(`unknown workload "${w}" (expected one of ${Object.keys(WORKLOADS).join(', ')})`);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const config: BenchmarkConfig = {
|
|
95
|
+
records: values.records ? Number(values.records) : preset.records,
|
|
96
|
+
opsPerWorkload,
|
|
97
|
+
concurrency,
|
|
98
|
+
loadConcurrency: values['load-concurrency'] ? Number(values['load-concurrency']) : concurrency,
|
|
99
|
+
fieldCount: Number(values.fields),
|
|
100
|
+
fieldLength: Number(values['field-length']),
|
|
101
|
+
maxScanLength: Number(values['scan-max']),
|
|
102
|
+
warmupOps: values.warmup ? Number(values.warmup) : Math.min(opsPerWorkload, 20_000),
|
|
103
|
+
workloads,
|
|
104
|
+
distribution: values.distribution as DistributionName | undefined,
|
|
105
|
+
table: 'usertable',
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
config,
|
|
110
|
+
scale: values.scale as string,
|
|
111
|
+
engine: values.engine as string,
|
|
112
|
+
threads: Number(values.threads),
|
|
113
|
+
nodeCount: Number(values.nodes),
|
|
114
|
+
out: values.out as string,
|
|
115
|
+
startupTimeoutMs: Number(values['startup-timeout']),
|
|
116
|
+
settleMs: Number(values['settle-ms']),
|
|
117
|
+
profile: values.profile as boolean,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function keyWidth(config: BenchmarkConfig): number {
|
|
122
|
+
return Math.max(10, String(config.records + config.opsPerWorkload).length);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface WorkloadResult {
|
|
126
|
+
name: string;
|
|
127
|
+
description: string;
|
|
128
|
+
distribution: DistributionName;
|
|
129
|
+
ops: number;
|
|
130
|
+
errors: number;
|
|
131
|
+
elapsedMs: number;
|
|
132
|
+
throughput: number;
|
|
133
|
+
latency: Partial<Record<OperationType, LatencyStats>>;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export interface BenchmarkResults {
|
|
137
|
+
meta: Record<string, unknown>;
|
|
138
|
+
config: BenchmarkConfig & { engine: string; threads: number; nodeCount: number; baseUrls: string[] };
|
|
139
|
+
load: { ops: number; errors: number; elapsedMs: number; throughput: number };
|
|
140
|
+
workloads: WorkloadResult[];
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function gitInfo(): { commit: string; branch: string } {
|
|
144
|
+
const read = (args: string[]): string => {
|
|
145
|
+
try {
|
|
146
|
+
return execFileSync('git', args, { encoding: 'utf8' }).trim();
|
|
147
|
+
} catch {
|
|
148
|
+
return 'unknown';
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
return { commit: read(['rev-parse', 'HEAD']), branch: read(['rev-parse', '--abbrev-ref', 'HEAD']) };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Runs the load phase then each selected workload against the given base URLs
|
|
156
|
+
* (one for single-node, several for a cluster — requests round-robin across
|
|
157
|
+
* them). Returns the assembled results; does not manage instance lifecycle.
|
|
158
|
+
*/
|
|
159
|
+
export async function runBenchmark(
|
|
160
|
+
baseUrls: string[],
|
|
161
|
+
options: ParsedOptions,
|
|
162
|
+
extraMeta: Record<string, unknown> = {}
|
|
163
|
+
): Promise<BenchmarkResults> {
|
|
164
|
+
const { config } = options;
|
|
165
|
+
const width = keyWidth(config);
|
|
166
|
+
const shape = { fieldCount: config.fieldCount, fieldLength: config.fieldLength };
|
|
167
|
+
const executor = createRestExecutor({
|
|
168
|
+
baseUrls,
|
|
169
|
+
table: config.table,
|
|
170
|
+
maxSockets: Math.max(config.concurrency, config.loadConcurrency),
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
const log = (msg: string) => process.stdout.write(`${msg}\n`);
|
|
174
|
+
|
|
175
|
+
try {
|
|
176
|
+
// --- Load phase: insert `records` records with sequential keys ---
|
|
177
|
+
log(`\n[load] inserting ${config.records.toLocaleString()} records (concurrency ${config.loadConcurrency})...`);
|
|
178
|
+
const loadKeys = new KeyState({
|
|
179
|
+
distribution: 'uniform',
|
|
180
|
+
initialKeyCount: 0,
|
|
181
|
+
keyWidth: width,
|
|
182
|
+
shape,
|
|
183
|
+
maxScanLength: config.maxScanLength,
|
|
184
|
+
});
|
|
185
|
+
const loadResult = await runOperations({
|
|
186
|
+
opCount: config.records,
|
|
187
|
+
concurrency: config.loadConcurrency,
|
|
188
|
+
mix: { insert: 1 },
|
|
189
|
+
executor,
|
|
190
|
+
keys: loadKeys,
|
|
191
|
+
onProgress: (done, total) => log(` loaded ${done.toLocaleString()} / ${total.toLocaleString()}`),
|
|
192
|
+
});
|
|
193
|
+
log(
|
|
194
|
+
`[load] ${loadResult.throughput.toFixed(0)} records/sec, ${loadResult.errors} errors, ${(loadResult.elapsedMs / 1000).toFixed(1)}s`
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
// --- Settle: let async replication converge before reading (cluster runs) ---
|
|
198
|
+
if (options.settleMs > 0) {
|
|
199
|
+
log(`[settle] waiting ${options.settleMs}ms for replication to converge...`);
|
|
200
|
+
await delay(options.settleMs);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// --- Optional warmup (discarded) ---
|
|
204
|
+
if (config.warmupOps > 0) {
|
|
205
|
+
log(`[warmup] ${config.warmupOps.toLocaleString()} read ops (discarded)...`);
|
|
206
|
+
await runOperations({
|
|
207
|
+
opCount: config.warmupOps,
|
|
208
|
+
concurrency: config.concurrency,
|
|
209
|
+
mix: { read: 1 },
|
|
210
|
+
executor,
|
|
211
|
+
keys: new KeyState({
|
|
212
|
+
distribution: 'zipfian',
|
|
213
|
+
initialKeyCount: config.records,
|
|
214
|
+
keyWidth: width,
|
|
215
|
+
shape,
|
|
216
|
+
maxScanLength: config.maxScanLength,
|
|
217
|
+
}),
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// --- Run phase: each selected workload against the loaded dataset ---
|
|
222
|
+
const workloadResults: WorkloadResult[] = [];
|
|
223
|
+
for (const name of config.workloads) {
|
|
224
|
+
const spec = WORKLOADS[name];
|
|
225
|
+
const distribution = config.distribution ?? spec.distribution;
|
|
226
|
+
log(`\n[workload ${name}] ${spec.description} — ${distribution}, ${config.opsPerWorkload.toLocaleString()} ops`);
|
|
227
|
+
const keys = new KeyState({
|
|
228
|
+
distribution,
|
|
229
|
+
initialKeyCount: config.records,
|
|
230
|
+
keyWidth: width,
|
|
231
|
+
shape,
|
|
232
|
+
maxScanLength: config.maxScanLength,
|
|
233
|
+
});
|
|
234
|
+
const result: PhaseResult = await runOperations({
|
|
235
|
+
opCount: config.opsPerWorkload,
|
|
236
|
+
concurrency: config.concurrency,
|
|
237
|
+
mix: spec.mix,
|
|
238
|
+
executor,
|
|
239
|
+
keys,
|
|
240
|
+
onProgress: (done, total) => log(` ${name}: ${done.toLocaleString()} / ${total.toLocaleString()}`),
|
|
241
|
+
});
|
|
242
|
+
log(`[workload ${name}] ${result.throughput.toFixed(0)} ops/sec, ${result.errors} errors`);
|
|
243
|
+
workloadResults.push({
|
|
244
|
+
name,
|
|
245
|
+
description: spec.description,
|
|
246
|
+
distribution,
|
|
247
|
+
ops: result.ops,
|
|
248
|
+
errors: result.errors,
|
|
249
|
+
elapsedMs: result.elapsedMs,
|
|
250
|
+
throughput: result.throughput,
|
|
251
|
+
latency: result.latency,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const { commit, branch } = gitInfo();
|
|
256
|
+
return {
|
|
257
|
+
meta: {
|
|
258
|
+
timestamp: new Date().toISOString(),
|
|
259
|
+
gitCommit: commit,
|
|
260
|
+
gitBranch: branch,
|
|
261
|
+
nodeVersion: process.version,
|
|
262
|
+
platform: `${process.platform}-${process.arch}`,
|
|
263
|
+
...extraMeta,
|
|
264
|
+
},
|
|
265
|
+
config: {
|
|
266
|
+
...config,
|
|
267
|
+
engine: options.engine,
|
|
268
|
+
threads: options.threads,
|
|
269
|
+
nodeCount: baseUrls.length,
|
|
270
|
+
baseUrls,
|
|
271
|
+
},
|
|
272
|
+
load: {
|
|
273
|
+
ops: loadResult.ops,
|
|
274
|
+
errors: loadResult.errors,
|
|
275
|
+
elapsedMs: loadResult.elapsedMs,
|
|
276
|
+
throughput: loadResult.throughput,
|
|
277
|
+
},
|
|
278
|
+
workloads: workloadResults,
|
|
279
|
+
};
|
|
280
|
+
} finally {
|
|
281
|
+
executor.close();
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export async function writeResults(results: BenchmarkResults, outDir: string, label: string): Promise<string> {
|
|
286
|
+
await mkdir(outDir, { recursive: true });
|
|
287
|
+
const stamp = results.meta.timestamp as string;
|
|
288
|
+
const file = join(outDir, `ycsb-${label}-${stamp.replace(/[:.]/g, '-')}.json`);
|
|
289
|
+
await writeFile(file, JSON.stringify(results, null, 2));
|
|
290
|
+
await writeFile(join(outDir, `ycsb-${label}-latest.json`), JSON.stringify(results, null, 2));
|
|
291
|
+
return file;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function fmt(n: number): string {
|
|
295
|
+
return n.toFixed(2).padStart(9);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export function printReport(results: BenchmarkResults): void {
|
|
299
|
+
const lines: string[] = [];
|
|
300
|
+
lines.push('');
|
|
301
|
+
lines.push('='.repeat(78));
|
|
302
|
+
lines.push(
|
|
303
|
+
`YCSB results — ${results.config.nodeCount} node(s), threads.count=${results.config.threads}, ${results.config.engine}`
|
|
304
|
+
);
|
|
305
|
+
lines.push(
|
|
306
|
+
`records=${results.config.records.toLocaleString()} ops/workload=${results.config.opsPerWorkload.toLocaleString()} concurrency=${results.config.concurrency}`
|
|
307
|
+
);
|
|
308
|
+
lines.push('='.repeat(78));
|
|
309
|
+
lines.push(`load: ${results.load.throughput.toFixed(0)} records/sec (${results.load.errors} errors)`);
|
|
310
|
+
lines.push('');
|
|
311
|
+
lines.push(
|
|
312
|
+
`${'workload'.padEnd(10)}${'op'.padEnd(8)}${'thrput/s'.padStart(10)}${'p50 ms'.padStart(10)}${'p95 ms'.padStart(10)}${'p99 ms'.padStart(10)}${'max ms'.padStart(10)}`
|
|
313
|
+
);
|
|
314
|
+
lines.push('-'.repeat(78));
|
|
315
|
+
for (const wl of results.workloads) {
|
|
316
|
+
const header = `${wl.name.padEnd(10)}`;
|
|
317
|
+
const types = Object.keys(wl.latency) as OperationType[];
|
|
318
|
+
types.forEach((type, i) => {
|
|
319
|
+
const s = wl.latency[type]!;
|
|
320
|
+
const prefix = i === 0 ? header : ' '.repeat(10);
|
|
321
|
+
const thr = i === 0 ? wl.throughput.toFixed(0).padStart(10) : ''.padStart(10);
|
|
322
|
+
lines.push(`${prefix}${type.padEnd(8)}${thr}${fmt(s.p50)}${fmt(s.p95)}${fmt(s.p99)}${fmt(s.max)}`);
|
|
323
|
+
});
|
|
324
|
+
if (wl.errors > 0) lines.push(`${' '.repeat(10)}(${wl.errors} errors)`);
|
|
325
|
+
}
|
|
326
|
+
lines.push('='.repeat(78));
|
|
327
|
+
process.stdout.write(lines.join('\n') + '\n');
|
|
328
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* REST transport for the YCSB workload runner.
|
|
3
|
+
*
|
|
4
|
+
* Maps CRUD operations onto Harper's HTTP REST interface:
|
|
5
|
+
* read -> GET /<table>/<key>
|
|
6
|
+
* insert -> PUT /<table>/<key> (full record)
|
|
7
|
+
* update -> PUT /<table>/<key> (full record replace)
|
|
8
|
+
* rmw -> GET then PUT
|
|
9
|
+
* scan -> GET /<table>/?id>=<startKey>&limit(<count>)
|
|
10
|
+
*
|
|
11
|
+
* Uses node:http(s) with a keep-alive agent (no external dependency) so the
|
|
12
|
+
* client's connection pool is sized to the configured concurrency and never
|
|
13
|
+
* becomes the bottleneck. Across multiple base URLs it round-robins requests,
|
|
14
|
+
* which is how the cluster benchmark spreads load over nodes.
|
|
15
|
+
*/
|
|
16
|
+
import http from 'node:http';
|
|
17
|
+
import https from 'node:https';
|
|
18
|
+
import { Buffer } from 'node:buffer';
|
|
19
|
+
import type { OpExecutor } from './workload.mts';
|
|
20
|
+
|
|
21
|
+
interface Endpoint {
|
|
22
|
+
lib: typeof http | typeof https;
|
|
23
|
+
hostname: string;
|
|
24
|
+
port: number;
|
|
25
|
+
agent: http.Agent;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface RestClientOptions {
|
|
29
|
+
baseUrls: string[];
|
|
30
|
+
table: string;
|
|
31
|
+
maxSockets: number;
|
|
32
|
+
auth?: { username: string; password: string };
|
|
33
|
+
/** Per-request timeout; a hung socket otherwise blocks a closed-loop worker forever. */
|
|
34
|
+
requestTimeoutMs?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface RestExecutor extends OpExecutor {
|
|
38
|
+
close(): void;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function createRestExecutor(options: RestClientOptions): RestExecutor {
|
|
42
|
+
const { table } = options;
|
|
43
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? 30_000;
|
|
44
|
+
const authHeader = options.auth
|
|
45
|
+
? 'Basic ' + Buffer.from(`${options.auth.username}:${options.auth.password}`).toString('base64')
|
|
46
|
+
: undefined;
|
|
47
|
+
|
|
48
|
+
const endpoints: Endpoint[] = options.baseUrls.map((raw) => {
|
|
49
|
+
const url = new URL(raw);
|
|
50
|
+
const secure = url.protocol === 'https:';
|
|
51
|
+
const lib = secure ? https : http;
|
|
52
|
+
return {
|
|
53
|
+
lib,
|
|
54
|
+
hostname: url.hostname,
|
|
55
|
+
port: Number(url.port) || (secure ? 443 : 80),
|
|
56
|
+
agent: new lib.Agent({ keepAlive: true, maxSockets: options.maxSockets, maxFreeSockets: options.maxSockets }),
|
|
57
|
+
};
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
let cursor = 0;
|
|
61
|
+
const nextEndpoint = (): Endpoint => endpoints[cursor++ % endpoints.length];
|
|
62
|
+
|
|
63
|
+
function send(method: string, path: string, body?: Buffer): Promise<Buffer> {
|
|
64
|
+
const endpoint = nextEndpoint();
|
|
65
|
+
const headers: Record<string, string> = {};
|
|
66
|
+
if (authHeader) headers.authorization = authHeader;
|
|
67
|
+
if (body) {
|
|
68
|
+
headers['content-type'] = 'application/json';
|
|
69
|
+
headers['content-length'] = String(body.length);
|
|
70
|
+
}
|
|
71
|
+
return new Promise<Buffer>((resolve, reject) => {
|
|
72
|
+
const req = endpoint.lib.request(
|
|
73
|
+
{ hostname: endpoint.hostname, port: endpoint.port, path, method, headers, agent: endpoint.agent },
|
|
74
|
+
(res) => {
|
|
75
|
+
const chunks: Buffer[] = [];
|
|
76
|
+
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
|
77
|
+
res.on('end', () => {
|
|
78
|
+
const status = res.statusCode ?? 0;
|
|
79
|
+
if (status >= 200 && status < 300) {
|
|
80
|
+
resolve(Buffer.concat(chunks));
|
|
81
|
+
} else {
|
|
82
|
+
reject(new Error(`${method} ${path} -> ${status}: ${Buffer.concat(chunks).toString().slice(0, 200)}`));
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
res.on('error', reject);
|
|
86
|
+
}
|
|
87
|
+
);
|
|
88
|
+
req.on('error', reject);
|
|
89
|
+
req.setTimeout(requestTimeoutMs, () =>
|
|
90
|
+
req.destroy(new Error(`${method} ${path} timed out after ${requestTimeoutMs}ms`))
|
|
91
|
+
);
|
|
92
|
+
if (body) req.write(body);
|
|
93
|
+
req.end();
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const recordPath = (key: string): string => `/${table}/${key}`;
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
async read(key: string): Promise<void> {
|
|
101
|
+
await send('GET', recordPath(key));
|
|
102
|
+
},
|
|
103
|
+
async insert(key: string, record: Record<string, string>): Promise<void> {
|
|
104
|
+
await send('PUT', recordPath(key), Buffer.from(JSON.stringify(record)));
|
|
105
|
+
},
|
|
106
|
+
async update(key: string, record: Record<string, string>): Promise<void> {
|
|
107
|
+
await send('PUT', recordPath(key), Buffer.from(JSON.stringify(record)));
|
|
108
|
+
},
|
|
109
|
+
async readModifyWrite(key: string, record: Record<string, string>): Promise<void> {
|
|
110
|
+
await send('GET', recordPath(key));
|
|
111
|
+
await send('PUT', recordPath(key), Buffer.from(JSON.stringify(record)));
|
|
112
|
+
},
|
|
113
|
+
async scan(startKey: string, count: number): Promise<void> {
|
|
114
|
+
await send('GET', `/${table}/?id>=${startKey}&limit(${count})`);
|
|
115
|
+
},
|
|
116
|
+
close(): void {
|
|
117
|
+
for (const endpoint of endpoints) endpoint.agent.destroy();
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YCSB-style CRUD load test against a single local Harper (core) instance.
|
|
3
|
+
*
|
|
4
|
+
* Boots one Harper instance via @harperfast/integration-testing with the
|
|
5
|
+
* `usertable` app pre-installed, then drives the YCSB load + run phases over
|
|
6
|
+
* the HTTP REST interface and writes JSON results.
|
|
7
|
+
*
|
|
8
|
+
* Build Harper first (npm run build), then run e.g.:
|
|
9
|
+
* node benchmarks/ycsb/run-single-node.mts --scale=standard
|
|
10
|
+
* node benchmarks/ycsb/run-single-node.mts --scale=quick --workloads=C,A
|
|
11
|
+
* node benchmarks/ycsb/run-single-node.mts --engine=lmdb --threads=4
|
|
12
|
+
*
|
|
13
|
+
* See harness.mts (parseOptions) for the full flag list.
|
|
14
|
+
*/
|
|
15
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
16
|
+
import { join } from 'node:path';
|
|
17
|
+
import { mkdir } from 'node:fs/promises';
|
|
18
|
+
import { createHarperContext, setupHarperWithFixture, teardownHarper } from '@harperfast/integration-testing';
|
|
19
|
+
import { parseOptions, runBenchmark, writeResults, printReport } from './harness.mts';
|
|
20
|
+
|
|
21
|
+
const REPO_ROOT = join(import.meta.dirname, '..', '..');
|
|
22
|
+
const HARPER_BIN = join(REPO_ROOT, 'dist', 'bin', 'harper.js');
|
|
23
|
+
const APP_DIR = join(import.meta.dirname, 'app');
|
|
24
|
+
|
|
25
|
+
async function waitForRoute(url: string, deadlineMs: number): Promise<void> {
|
|
26
|
+
const deadline = Date.now() + deadlineMs;
|
|
27
|
+
while (Date.now() < deadline) {
|
|
28
|
+
try {
|
|
29
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
|
30
|
+
await res.body?.cancel(); // always drain so the socket is freed
|
|
31
|
+
// Require 2xx: after startup the HTTP stack is up but the usertable route may not be
|
|
32
|
+
// registered yet, and a 404 from the router would falsely pass a < 500 check.
|
|
33
|
+
if (res.status >= 200 && res.status < 300) return;
|
|
34
|
+
} catch {
|
|
35
|
+
// not accepting connections yet (or this probe timed out)
|
|
36
|
+
}
|
|
37
|
+
await delay(250);
|
|
38
|
+
}
|
|
39
|
+
throw new Error(`timed out waiting for ${url}`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function main(): Promise<void> {
|
|
43
|
+
const options = parseOptions(process.argv.slice(2));
|
|
44
|
+
const ctx = createHarperContext('ycsb-single-node');
|
|
45
|
+
|
|
46
|
+
// In --profile mode, run the HTTP server on the main thread (threads.count=0)
|
|
47
|
+
// so a single NODE_OPTIONS --cpu-prof captures request handling — Harper's
|
|
48
|
+
// worker threads use a fixed execArgv that --cpu-prof can't reach.
|
|
49
|
+
const threads = options.profile ? 0 : options.threads;
|
|
50
|
+
// This config is passed as HARPER_SET_CONFIG, which takes precedence over (and filters out)
|
|
51
|
+
// the framework's hard-coded CLI args — so threads.count and logging.level here override the
|
|
52
|
+
// framework defaults of --THREADS_COUNT=1 and --LOGGING_LEVEL=debug. The latter matters:
|
|
53
|
+
// debug logging under load would be a major measurement confound.
|
|
54
|
+
const config: Record<string, unknown> = {
|
|
55
|
+
threads: { count: threads },
|
|
56
|
+
analytics: { aggregatePeriod: -1 }, // analytics aggregation is noisy under load
|
|
57
|
+
logging: { level: 'warn' },
|
|
58
|
+
};
|
|
59
|
+
// Set the engine explicitly (not only when non-default) so the run is pinned to the
|
|
60
|
+
// requested engine even if Harper's default changes, matching the reported config.
|
|
61
|
+
const env: Record<string, string> = { HARPER_STORAGE_ENGINE: options.engine };
|
|
62
|
+
let profileDir: string | undefined;
|
|
63
|
+
if (options.profile) {
|
|
64
|
+
profileDir = join(options.out, 'profile');
|
|
65
|
+
await mkdir(profileDir, { recursive: true });
|
|
66
|
+
env.NODE_OPTIONS = `--cpu-prof --cpu-prof-dir=${profileDir}`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
console.log(
|
|
70
|
+
`Starting Harper (threads.count=${threads}, engine=${options.engine}${options.profile ? ', profiling' : ''})...`
|
|
71
|
+
);
|
|
72
|
+
await setupHarperWithFixture(ctx, APP_DIR, {
|
|
73
|
+
harperBinPath: HARPER_BIN,
|
|
74
|
+
config,
|
|
75
|
+
env,
|
|
76
|
+
startupTimeoutMs: options.startupTimeoutMs,
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
const { httpURL } = ctx.harper;
|
|
80
|
+
try {
|
|
81
|
+
await waitForRoute(`${httpURL}/${options.config.table}/`, 30_000);
|
|
82
|
+
console.log(`Harper ready at ${httpURL}`);
|
|
83
|
+
const results = await runBenchmark([httpURL], options);
|
|
84
|
+
const file = await writeResults(results, options.out, 'single-node');
|
|
85
|
+
printReport(results);
|
|
86
|
+
console.log(`\nResults written to ${file}`);
|
|
87
|
+
} finally {
|
|
88
|
+
await teardownHarper(ctx);
|
|
89
|
+
console.log('Harper stopped.');
|
|
90
|
+
if (profileDir) console.log(`CPU profile(s) in ${profileDir}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
main().catch((error) => {
|
|
95
|
+
console.error(error);
|
|
96
|
+
process.exit(1);
|
|
97
|
+
});
|