@indigoai-us/hq-cloud 6.15.11 → 6.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +8 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/share.d.ts +49 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +57 -3
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +131 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +10 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/personal-vault.d.ts +13 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +47 -13
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +76 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/sync/lease-client.d.ts +1 -0
- package/dist/sync/lease-client.d.ts.map +1 -1
- package/dist/sync/lease-client.js +16 -11
- package/dist/sync/lease-client.js.map +1 -1
- package/dist/sync/lease-client.test.js +3 -3
- package/dist/sync/lease-client.test.js.map +1 -1
- package/dist/sync/realtime-rollout.d.ts +2 -1
- package/dist/sync/realtime-rollout.d.ts.map +1 -1
- package/dist/sync/realtime-rollout.js +11 -5
- package/dist/sync/realtime-rollout.js.map +1 -1
- package/dist/sync/realtime-rollout.test.js +27 -4
- package/dist/sync/realtime-rollout.test.js.map +1 -1
- package/dist/sync/scope-inventory-client.test.js +13 -4
- package/dist/sync/scope-inventory-client.test.js.map +1 -1
- package/dist/sync-core.d.ts +58 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +133 -5
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.d.ts +2 -0
- package/dist/sync-core.test.d.ts.map +1 -0
- package/dist/sync-core.test.js +113 -0
- package/dist/sync-core.test.js.map +1 -0
- package/package.json +1 -1
|
@@ -4,6 +4,7 @@ import * as os from "node:os";
|
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
6
6
|
import { ScopeInventoryClient, disposeLocalMaterial, parseLocalMaterialPolicy, } from "./scope-inventory-client.js";
|
|
7
|
+
import { parseRealtimeInventory } from "./realtime-rollout.js";
|
|
7
8
|
const roots = [];
|
|
8
9
|
function root() {
|
|
9
10
|
const value = fs.mkdtempSync(path.join(os.tmpdir(), "scope-inventory-"));
|
|
@@ -34,8 +35,8 @@ afterEach(() => {
|
|
|
34
35
|
describe("ScopeInventoryClient", () => {
|
|
35
36
|
it("stops network work on reset, then permits legacy only after a fresh returning-device inventory", async () => {
|
|
36
37
|
const client = inventoryClient([
|
|
37
|
-
{ scopes: [{ scopeUid: "prs_1", scopeHandle: "h1", scopeRevision: "r1", state: "reset-required", resetId: "reset-1", snapshotDescriptorId: "snap-1" }] },
|
|
38
|
-
{ scopes: [{ scopeUid: "prs_1", scopeHandle: "h2", scopeRevision: "r2", state: "legacy", v1Allowed: true }] },
|
|
38
|
+
{ contractVersion: 2, scopes: [{ scopeUid: "prs_1", scopeHandle: "h1", scopeRevision: "r1", state: "reset-required", resetId: "reset-1", snapshotDescriptorId: "snap-1" }] },
|
|
39
|
+
{ contractVersion: 2, scopes: [{ scopeUid: "prs_1", scopeHandle: "h2", scopeRevision: "r2", state: "legacy", v1Allowed: true }] },
|
|
39
40
|
]);
|
|
40
41
|
const gate = new ScopeInventoryClient({ inventory: client, request: request() });
|
|
41
42
|
await expect(gate.refresh("prs_1")).resolves.toMatchObject({ network: "stop", state: "reset-required", reason: "reset-required" });
|
|
@@ -44,8 +45,8 @@ describe("ScopeInventoryClient", () => {
|
|
|
44
45
|
});
|
|
45
46
|
it("fails closed for a revoked or absent scope and retains unknown local policy", async () => {
|
|
46
47
|
const client = inventoryClient([
|
|
47
|
-
{ scopes: [{ scopeUid: "cmp_1", scopeHandle: "h", scopeRevision: "r", state: "revoked", localMaterialPolicy: "future-policy" }] },
|
|
48
|
-
{ scopes: [] },
|
|
48
|
+
{ contractVersion: 2, scopes: [{ scopeUid: "cmp_1", scopeHandle: "h", scopeRevision: "r", state: "revoked", localMaterialPolicy: "future-policy" }] },
|
|
49
|
+
{ contractVersion: 2, scopes: [] },
|
|
49
50
|
]);
|
|
50
51
|
const gate = new ScopeInventoryClient({ inventory: client, request: request() });
|
|
51
52
|
await expect(gate.refresh("cmp_1")).resolves.toEqual(expect.objectContaining({
|
|
@@ -54,6 +55,14 @@ describe("ScopeInventoryClient", () => {
|
|
|
54
55
|
await expect(gate.refresh("cmp_1")).resolves.toEqual({ network: "stop", state: "absent", reason: "scope-absent" });
|
|
55
56
|
expect(parseLocalMaterialPolicy("not-yet-defined")).toBe("retain");
|
|
56
57
|
});
|
|
58
|
+
it("stops a parsed blocked grant with an explicitly unusable handle", async () => {
|
|
59
|
+
const client = inventoryClient([parseRealtimeInventory({
|
|
60
|
+
contractVersion: 2,
|
|
61
|
+
scopes: [{ scopeUid: "prs_1", scopeHandle: null, scopeRevision: "r1", state: "blocked", reasonCode: "SCOPE_HANDLE_MISSING" }],
|
|
62
|
+
})]);
|
|
63
|
+
const gate = new ScopeInventoryClient({ inventory: client, request: request() });
|
|
64
|
+
await expect(gate.refresh("prs_1")).resolves.toMatchObject({ network: "stop", state: "blocked", reason: "scope-blocked" });
|
|
65
|
+
});
|
|
57
66
|
});
|
|
58
67
|
describe("disposeLocalMaterial", () => {
|
|
59
68
|
it("deletes only a hash-matching file and fsyncs its parent", () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scope-inventory-client.test.js","sourceRoot":"","sources":["../../src/sync/scope-inventory-client.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE7D,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"scope-inventory-client.test.js","sourceRoot":"","sources":["../../src/sync/scope-inventory-client.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE7D,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,sBAAsB,EAAsD,MAAM,uBAAuB,CAAC;AAEnH,MAAM,KAAK,GAAa,EAAE,CAAC;AAE3B,SAAS,IAAI;IACX,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACzE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,SAAS,eAAe,CAAC,WAAgC;IACvD,OAAO;QACL,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,KAAK,EAAG,CAAC;KACf,CAAC;AACxC,CAAC;AAED,SAAS,OAAO;IACd,OAAO;QACL,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE,UAAU;QACpB,gBAAgB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAe,EAAE,QAAQ,EAAE,CAAU,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;QACrH,wBAAwB,EAAE,iBAAiB;QAC3C,gBAAgB,EAAE,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,CAAC,GAAG,EAAE;IACb,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAClG,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,gGAAgG,EAAE,KAAK,IAAI,EAAE;QAC9G,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC,EAAE;YAC5K,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE;SAClI,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAEjF,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACnI,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QACzI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,eAAe,EAAE,CAAC,EAAE;YACrJ,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAEjF,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAC3E,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,mBAAmB,EAAE,QAAQ;SAC1F,CAAC,CAAC,CAAC;QACJ,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QACnH,MAAM,CAAC,wBAAwB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,sBAAsB,CAAC;gBACrD,eAAe,EAAE,CAAC;gBAClB,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,sBAAsB,EAAE,CAAC;aAC9H,CAAC,CAAC,CAAC,CAAC;QACL,MAAM,IAAI,GAAG,IAAI,oBAAoB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAEjF,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;IAC7H,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEnC,MAAM,CAAC,oBAAoB,CAAC;YAC1B,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,YAAY;YAC1B,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC;YACtC,MAAM,EAAE,wBAAwB;SACjC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,GAAG,EAAE;QAC5F,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;QACrB,MAAM,cAAc,GAAG,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,oBAAoB,CAAC;YAClC,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,YAAY;YAC1B,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC;YACtC,MAAM,EAAE,wBAAwB;YAChC,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,aAAa;YACtB,MAAM,EAAE,MAAM,CAAC,uBAAuB,CAAC;YACvC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC;SACxD,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACzG,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/sync-core.d.ts
CHANGED
|
@@ -5,7 +5,64 @@
|
|
|
5
5
|
* Forbidden).
|
|
6
6
|
*/
|
|
7
7
|
export declare function isAccessDenied(err: unknown): boolean;
|
|
8
|
-
|
|
8
|
+
/** Host resource snapshot used to decide the default transfer fan-out. */
|
|
9
|
+
export interface HostLoadProbe {
|
|
10
|
+
/** Logical CPU count (0 when unknown). */
|
|
11
|
+
cores: number;
|
|
12
|
+
/** 1-minute load average (0 on platforms without it, e.g. Windows). */
|
|
13
|
+
load1: number;
|
|
14
|
+
/**
|
|
15
|
+
* Memory *available to applications* as a fraction of total (0..1). This is
|
|
16
|
+
* NOT `os.freemem() / os.totalmem()`: on macOS and Linux the OS keeps most
|
|
17
|
+
* "free" RAM as reclaimable file cache, so raw free-page counts report a
|
|
18
|
+
* near-full machine as memory-starved. `readAvailableMemoryFraction`
|
|
19
|
+
* measures available memory per-platform instead.
|
|
20
|
+
*/
|
|
21
|
+
availableMemFraction: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Measure memory *available to applications* as a fraction of total RAM.
|
|
25
|
+
*
|
|
26
|
+
* `os.freemem()` counts only unused pages; both macOS and Linux hold most idle
|
|
27
|
+
* RAM as reclaimable cache, so it drastically understates what an app can get
|
|
28
|
+
* (an idle 16 GB Mac reports ~8% "free" while ~75% is actually available).
|
|
29
|
+
* Reading it as pressure would pin every pass to the busy tier. We instead read
|
|
30
|
+
* the platform's available figure; any failure falls back optimistically (1.0)
|
|
31
|
+
* so a bad probe never throttles — the upload byte budget is the real
|
|
32
|
+
* memory-safety net.
|
|
33
|
+
*/
|
|
34
|
+
export declare function readAvailableMemoryFraction(): number;
|
|
35
|
+
/** Read the current host load. Pure I/O; callers inject a probe in tests. */
|
|
36
|
+
export declare function readHostLoad(): HostLoadProbe;
|
|
37
|
+
/**
|
|
38
|
+
* True when the host is already under meaningful CPU or memory load from other
|
|
39
|
+
* processes. `cores === 0` (unknown) disables the CPU signal rather than
|
|
40
|
+
* dividing by zero; `load1 === 0` (Windows has no load average) likewise leaves
|
|
41
|
+
* the CPU signal inert, so those hosts fall back to the memory signal only.
|
|
42
|
+
*/
|
|
43
|
+
export declare function machineAlreadyBusy(probe: HostLoadProbe): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Concurrency for the per-file transfer pool (uploads + downloads).
|
|
46
|
+
*
|
|
47
|
+
* An explicit, valid `HQ_SYNC_TRANSFER_CONCURRENCY` always wins. Otherwise the
|
|
48
|
+
* default adapts to the host: 64-way normally, 32-way when the machine is
|
|
49
|
+
* already busy (see `machineAlreadyBusy`). Callers capture the probe at pass
|
|
50
|
+
* entry — BEFORE the walk/hash pass — and inject it, so the reading reflects
|
|
51
|
+
* other processes rather than sync's own load; tests inject a synthetic probe.
|
|
52
|
+
*/
|
|
53
|
+
export declare function resolveTransferConcurrency(probe?: HostLoadProbe): number;
|
|
54
|
+
/**
|
|
55
|
+
* Total in-flight upload-body bytes the upload pool may hold at once. Because
|
|
56
|
+
* `uploadFile` buffers whole files, this — not the worker count — is what
|
|
57
|
+
* bounds sync's peak memory. Sized from AVAILABLE memory (the pass-entry
|
|
58
|
+
* probe's `availableMemFraction`), so a memory-tight host gets a small budget
|
|
59
|
+
* that cannot exhaust what is left while an idle host is capped at 1 GiB. No
|
|
60
|
+
* hard floor: a tiny budget is safe because the pool always admits one item
|
|
61
|
+
* when nothing is in flight, so a single oversized file still uploads. Override
|
|
62
|
+
* for tuning/tests with `HQ_SYNC_UPLOAD_BYTE_BUDGET_MB` (absolute, fractional
|
|
63
|
+
* MB accepted).
|
|
64
|
+
*/
|
|
65
|
+
export declare function resolveUploadByteBudget(availableMemFraction?: number, totalMemBytes?: number): number;
|
|
9
66
|
/**
|
|
10
67
|
* Resolve active company from .hq/config.json.
|
|
11
68
|
*/
|
package/dist/sync-core.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-core.d.ts","sourceRoot":"","sources":["../src/sync-core.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sync-core.d.ts","sourceRoot":"","sources":["../src/sync-core.ts"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAMpD;AAED,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAyBD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,IAAI,MAAM,CA+BpD;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,IAAI,aAAa,CAI5C;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAKhE;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAgBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACrC,oBAAoB,GAAE,MAAsC,EAC5D,aAAa,GAAE,MAAsB,GACpC,MAAM,CASR;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAWvE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE;IAAE,YAAY,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAC5C,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/C,OAAO,CAMT"}
|
package/dist/sync-core.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { execSync } from "child_process";
|
|
1
2
|
import * as fs from "fs";
|
|
3
|
+
import * as os from "os";
|
|
2
4
|
import * as path from "path";
|
|
3
5
|
import { normalizeEtag } from "./journal.js";
|
|
4
6
|
/**
|
|
@@ -14,12 +16,138 @@ export function isAccessDenied(err) {
|
|
|
14
16
|
}
|
|
15
17
|
return false;
|
|
16
18
|
}
|
|
17
|
-
|
|
19
|
+
// Default transfer fan-out is high because transfers are network-round-trip
|
|
20
|
+
// bound, not CPU bound: 64-way shortened a ~459k-file first sync from ~7 to
|
|
21
|
+
// ~35+ files/sec in the 2026-08 personal-vault soak. We step down to 32 only
|
|
22
|
+
// when the host is ALREADY busy from other work at pass start (probe captured
|
|
23
|
+
// before sync walks/hashes, so it reflects other processes, not our own load),
|
|
24
|
+
// so sync does not compound CPU or memory pressure the user is already under.
|
|
25
|
+
const DEFAULT_TRANSFER_CONCURRENCY = 64;
|
|
26
|
+
const BUSY_TRANSFER_CONCURRENCY = 32;
|
|
27
|
+
// "Already busy" thresholds, measured before sync adds its own load.
|
|
28
|
+
const HIGH_LOAD_PER_CORE = 0.5; // >=50% of cores runnable from other work
|
|
29
|
+
const MIN_AVAILABLE_MEM_FRACTION = 0.25; // <25% RAM available => memory-tight host
|
|
30
|
+
// Worst-case in-flight upload-body memory. `uploadFile` reads each file fully
|
|
31
|
+
// into a Buffer, so a count-only pool of 64 could hold dozens of large files
|
|
32
|
+
// (a personal vault carries 184 MB session transcripts) at once. The upload
|
|
33
|
+
// pool caps total in-flight bytes to this budget REGARDLESS of worker count;
|
|
34
|
+
// see resolveUploadByteBudget.
|
|
35
|
+
const UPLOAD_BYTE_BUDGET_CAP = 1024 * 1024 * 1024; // 1 GiB
|
|
36
|
+
// Fraction of AVAILABLE memory (not total) to allot to in-flight upload bodies;
|
|
37
|
+
// the remaining ~75% is headroom. Available-based so a memory-tight host gets a
|
|
38
|
+
// proportionally smaller budget instead of one that could exhaust what's left.
|
|
39
|
+
const UPLOAD_BYTE_BUDGET_AVAIL_FRACTION = 0.25;
|
|
40
|
+
/**
|
|
41
|
+
* Measure memory *available to applications* as a fraction of total RAM.
|
|
42
|
+
*
|
|
43
|
+
* `os.freemem()` counts only unused pages; both macOS and Linux hold most idle
|
|
44
|
+
* RAM as reclaimable cache, so it drastically understates what an app can get
|
|
45
|
+
* (an idle 16 GB Mac reports ~8% "free" while ~75% is actually available).
|
|
46
|
+
* Reading it as pressure would pin every pass to the busy tier. We instead read
|
|
47
|
+
* the platform's available figure; any failure falls back optimistically (1.0)
|
|
48
|
+
* so a bad probe never throttles — the upload byte budget is the real
|
|
49
|
+
* memory-safety net.
|
|
50
|
+
*/
|
|
51
|
+
export function readAvailableMemoryFraction() {
|
|
52
|
+
const total = os.totalmem();
|
|
53
|
+
if (total <= 0)
|
|
54
|
+
return 1;
|
|
55
|
+
try {
|
|
56
|
+
if (process.platform === "linux") {
|
|
57
|
+
const meminfo = fs.readFileSync("/proc/meminfo", "utf8");
|
|
58
|
+
const m = meminfo.match(/^MemAvailable:\s+(\d+)\s*kB/m);
|
|
59
|
+
if (m)
|
|
60
|
+
return Math.min(1, (Number(m[1]) * 1024) / total);
|
|
61
|
+
return os.freemem() / total; // pre-3.14 kernels lack MemAvailable
|
|
62
|
+
}
|
|
63
|
+
if (process.platform === "darwin") {
|
|
64
|
+
const out = execSync("vm_stat", { encoding: "utf8", timeout: 2000 });
|
|
65
|
+
const pageSize = Number(out.match(/page size of (\d+) bytes/)?.[1] ?? 4096);
|
|
66
|
+
const pagesOf = (label) => Number(out.match(new RegExp(`${label}:\\s+(\\d+)\\.`))?.[1] ?? 0);
|
|
67
|
+
// Available ~= free + inactive + speculative + purgeable (all reclaimable
|
|
68
|
+
// without paging to disk) — mirrors Activity Monitor's "available".
|
|
69
|
+
const availablePages = pagesOf("Pages free") +
|
|
70
|
+
pagesOf("Pages inactive") +
|
|
71
|
+
pagesOf("Pages speculative") +
|
|
72
|
+
pagesOf("Pages purgeable");
|
|
73
|
+
const avail = availablePages * pageSize;
|
|
74
|
+
return avail > 0 ? Math.min(1, avail / total) : os.freemem() / total;
|
|
75
|
+
}
|
|
76
|
+
// Windows and other platforms: os.freemem() reports actual available
|
|
77
|
+
// physical memory (no reclaimable-cache accounting gap), so use it directly.
|
|
78
|
+
return os.freemem() / total;
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return 1; // fail open: never let a probe error wrongly step down
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Read the current host load. Pure I/O; callers inject a probe in tests. */
|
|
85
|
+
export function readHostLoad() {
|
|
86
|
+
const cores = os.cpus()?.length ?? 0;
|
|
87
|
+
const load1 = os.loadavg()?.[0] ?? 0;
|
|
88
|
+
return { cores, load1, availableMemFraction: readAvailableMemoryFraction() };
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* True when the host is already under meaningful CPU or memory load from other
|
|
92
|
+
* processes. `cores === 0` (unknown) disables the CPU signal rather than
|
|
93
|
+
* dividing by zero; `load1 === 0` (Windows has no load average) likewise leaves
|
|
94
|
+
* the CPU signal inert, so those hosts fall back to the memory signal only.
|
|
95
|
+
*/
|
|
96
|
+
export function machineAlreadyBusy(probe) {
|
|
97
|
+
const perCoreLoad = probe.cores > 0 ? probe.load1 / probe.cores : 0;
|
|
98
|
+
const cpuBusy = perCoreLoad >= HIGH_LOAD_PER_CORE;
|
|
99
|
+
const memoryTight = probe.availableMemFraction < MIN_AVAILABLE_MEM_FRACTION;
|
|
100
|
+
return cpuBusy || memoryTight;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Concurrency for the per-file transfer pool (uploads + downloads).
|
|
104
|
+
*
|
|
105
|
+
* An explicit, valid `HQ_SYNC_TRANSFER_CONCURRENCY` always wins. Otherwise the
|
|
106
|
+
* default adapts to the host: 64-way normally, 32-way when the machine is
|
|
107
|
+
* already busy (see `machineAlreadyBusy`). Callers capture the probe at pass
|
|
108
|
+
* entry — BEFORE the walk/hash pass — and inject it, so the reading reflects
|
|
109
|
+
* other processes rather than sync's own load; tests inject a synthetic probe.
|
|
110
|
+
*/
|
|
111
|
+
export function resolveTransferConcurrency(probe) {
|
|
18
112
|
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
19
|
-
if (raw
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
113
|
+
if (raw !== undefined && raw !== "") {
|
|
114
|
+
const parsed = Number.parseInt(raw, 10);
|
|
115
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
116
|
+
return parsed;
|
|
117
|
+
// Malformed value: fall through to the adaptive default (a strictly higher
|
|
118
|
+
// floor than the historical hard-coded 16).
|
|
119
|
+
}
|
|
120
|
+
let busy = false;
|
|
121
|
+
try {
|
|
122
|
+
busy = machineAlreadyBusy(probe ?? readHostLoad());
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
// A probe failure must never throttle throughput below the fast default.
|
|
126
|
+
busy = false;
|
|
127
|
+
}
|
|
128
|
+
return busy ? BUSY_TRANSFER_CONCURRENCY : DEFAULT_TRANSFER_CONCURRENCY;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Total in-flight upload-body bytes the upload pool may hold at once. Because
|
|
132
|
+
* `uploadFile` buffers whole files, this — not the worker count — is what
|
|
133
|
+
* bounds sync's peak memory. Sized from AVAILABLE memory (the pass-entry
|
|
134
|
+
* probe's `availableMemFraction`), so a memory-tight host gets a small budget
|
|
135
|
+
* that cannot exhaust what is left while an idle host is capped at 1 GiB. No
|
|
136
|
+
* hard floor: a tiny budget is safe because the pool always admits one item
|
|
137
|
+
* when nothing is in flight, so a single oversized file still uploads. Override
|
|
138
|
+
* for tuning/tests with `HQ_SYNC_UPLOAD_BYTE_BUDGET_MB` (absolute, fractional
|
|
139
|
+
* MB accepted).
|
|
140
|
+
*/
|
|
141
|
+
export function resolveUploadByteBudget(availableMemFraction = readAvailableMemoryFraction(), totalMemBytes = os.totalmem()) {
|
|
142
|
+
const raw = process.env.HQ_SYNC_UPLOAD_BYTE_BUDGET_MB;
|
|
143
|
+
if (raw !== undefined && raw !== "") {
|
|
144
|
+
const parsed = Number.parseFloat(raw);
|
|
145
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
146
|
+
return Math.floor(parsed * 1024 * 1024);
|
|
147
|
+
}
|
|
148
|
+
const availableBytes = Math.max(0, availableMemFraction) * totalMemBytes;
|
|
149
|
+
const scaled = Math.floor(availableBytes * UPLOAD_BYTE_BUDGET_AVAIL_FRACTION);
|
|
150
|
+
return Math.min(UPLOAD_BYTE_BUDGET_CAP, Math.max(1, scaled));
|
|
23
151
|
}
|
|
24
152
|
/**
|
|
25
153
|
* Resolve active company from .hq/config.json.
|
package/dist/sync-core.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-core.js","sourceRoot":"","sources":["../src/sync-core.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,GAAI,GAA0B,CAAC,IAAI,CAAC;QAC9C,OAAO,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,WAAW,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,0BAA0B;
|
|
1
|
+
{"version":3,"file":"sync-core.js","sourceRoot":"","sources":["../src/sync-core.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,GAAI,GAA0B,CAAC,IAAI,CAAC;QAC9C,OAAO,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,WAAW,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAkBD,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,8EAA8E;AAC9E,+EAA+E;AAC/E,8EAA8E;AAC9E,MAAM,4BAA4B,GAAG,EAAE,CAAC;AACxC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AACrC,qEAAqE;AACrE,MAAM,kBAAkB,GAAG,GAAG,CAAC,CAAC,0CAA0C;AAC1E,MAAM,0BAA0B,GAAG,IAAI,CAAC,CAAC,0CAA0C;AAEnF,8EAA8E;AAC9E,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,+BAA+B;AAC/B,MAAM,sBAAsB,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAC3D,gFAAgF;AAChF,gFAAgF;AAChF,+EAA+E;AAC/E,MAAM,iCAAiC,GAAG,IAAI,CAAC;AAE/C;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B;IACzC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC5B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YACxD,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACzD,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,qCAAqC;QACpE,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAC5E,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CACxC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACpE,0EAA0E;YAC1E,oEAAoE;YACpE,MAAM,cAAc,GAClB,OAAO,CAAC,YAAY,CAAC;gBACrB,OAAO,CAAC,gBAAgB,CAAC;gBACzB,OAAO,CAAC,mBAAmB,CAAC;gBAC5B,OAAO,CAAC,iBAAiB,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,cAAc,GAAG,QAAQ,CAAC;YACxC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC;QACvE,CAAC;QACD,qEAAqE;QACrE,6EAA6E;QAC7E,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC,CAAC,uDAAuD;IACnE,CAAC;AACH,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,YAAY;IAC1B,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,MAAM,IAAI,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,2BAA2B,EAAE,EAAE,CAAC;AAC/E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAoB;IACrD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,WAAW,IAAI,kBAAkB,CAAC;IAClD,MAAM,WAAW,GAAG,KAAK,CAAC,oBAAoB,GAAG,0BAA0B,CAAC;IAC5E,OAAO,OAAO,IAAI,WAAW,CAAC;AAChC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,KAAqB;IAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC;IACrD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QACzD,2EAA2E;QAC3E,4CAA4C;IAC9C,CAAC;IACD,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,kBAAkB,CAAC,KAAK,IAAI,YAAY,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,yEAAyE;QACzE,IAAI,GAAG,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,4BAA4B,CAAC;AACzE,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,uBAAuB,CACrC,uBAA+B,2BAA2B,EAAE,EAC5D,gBAAwB,EAAE,CAAC,QAAQ,EAAE;IAErC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC;IACtD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC,GAAG,aAAa,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,iCAAiC,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;IAC3D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;YAChE,OAAO,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,WAAW,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAA4C,EAC5C,KAAgD;IAEhD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;IACpD,OAAO,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC;AAClD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync-core.test.d.ts","sourceRoot":"","sources":["../src/sync-core.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
2
|
+
import { machineAlreadyBusy, resolveTransferConcurrency, resolveUploadByteBudget, } from "./sync-core.js";
|
|
3
|
+
// A host doing nothing else: a fraction of a core runnable, most RAM available.
|
|
4
|
+
const IDLE = { cores: 8, load1: 0.5, availableMemFraction: 0.6 };
|
|
5
|
+
// CPU already saturated by other work: per-core load 0.8 (>= 0.5 threshold).
|
|
6
|
+
const BUSY_CPU = { cores: 8, load1: 6.4, availableMemFraction: 0.6 };
|
|
7
|
+
// Memory already tight: 10% available (< 25% floor), CPU quiet.
|
|
8
|
+
const TIGHT_MEM = { cores: 8, load1: 0.2, availableMemFraction: 0.1 };
|
|
9
|
+
describe("resolveTransferConcurrency", () => {
|
|
10
|
+
const KEY = "HQ_SYNC_TRANSFER_CONCURRENCY";
|
|
11
|
+
let saved;
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
saved = process.env[KEY];
|
|
14
|
+
delete process.env[KEY];
|
|
15
|
+
});
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
if (saved === undefined)
|
|
18
|
+
delete process.env[KEY];
|
|
19
|
+
else
|
|
20
|
+
process.env[KEY] = saved;
|
|
21
|
+
});
|
|
22
|
+
it("defaults to 64-way on an otherwise-idle host", () => {
|
|
23
|
+
expect(resolveTransferConcurrency(IDLE)).toBe(64);
|
|
24
|
+
});
|
|
25
|
+
it("steps down to 32 when the host CPU is already busy", () => {
|
|
26
|
+
expect(resolveTransferConcurrency(BUSY_CPU)).toBe(32);
|
|
27
|
+
});
|
|
28
|
+
it("steps down to 32 when host available memory is already tight", () => {
|
|
29
|
+
expect(resolveTransferConcurrency(TIGHT_MEM)).toBe(32);
|
|
30
|
+
});
|
|
31
|
+
it("honors an explicit valid override regardless of host load", () => {
|
|
32
|
+
process.env[KEY] = "100";
|
|
33
|
+
expect(resolveTransferConcurrency(BUSY_CPU)).toBe(100);
|
|
34
|
+
process.env[KEY] = "4";
|
|
35
|
+
expect(resolveTransferConcurrency(IDLE)).toBe(4);
|
|
36
|
+
});
|
|
37
|
+
it("ignores a malformed/zero/negative override and uses the adaptive default", () => {
|
|
38
|
+
process.env[KEY] = "abc";
|
|
39
|
+
expect(resolveTransferConcurrency(IDLE)).toBe(64);
|
|
40
|
+
process.env[KEY] = "0";
|
|
41
|
+
expect(resolveTransferConcurrency(BUSY_CPU)).toBe(32);
|
|
42
|
+
process.env[KEY] = "-5";
|
|
43
|
+
expect(resolveTransferConcurrency(IDLE)).toBe(64);
|
|
44
|
+
});
|
|
45
|
+
it("reads real host load when no probe is injected (never throws, always positive)", () => {
|
|
46
|
+
delete process.env[KEY];
|
|
47
|
+
const n = resolveTransferConcurrency();
|
|
48
|
+
expect(n === 32 || n === 64).toBe(true);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe("machineAlreadyBusy", () => {
|
|
52
|
+
it("is false on an otherwise-idle host", () => {
|
|
53
|
+
expect(machineAlreadyBusy(IDLE)).toBe(false);
|
|
54
|
+
});
|
|
55
|
+
it("is true exactly at the per-core load threshold (0.5)", () => {
|
|
56
|
+
// cores 10, load1 5 => per-core 0.5
|
|
57
|
+
expect(machineAlreadyBusy({ cores: 10, load1: 5, availableMemFraction: 0.6 })).toBe(true);
|
|
58
|
+
});
|
|
59
|
+
it("is false just under the per-core load threshold", () => {
|
|
60
|
+
// cores 10, load1 4.9 => per-core 0.49
|
|
61
|
+
expect(machineAlreadyBusy({ cores: 10, load1: 4.9, availableMemFraction: 0.6 })).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
it("is true just under the available-memory floor (0.25)", () => {
|
|
64
|
+
expect(machineAlreadyBusy({ cores: 10, load1: 0, availableMemFraction: 0.249 })).toBe(true);
|
|
65
|
+
});
|
|
66
|
+
it("is false at the available-memory floor", () => {
|
|
67
|
+
expect(machineAlreadyBusy({ cores: 10, load1: 0, availableMemFraction: 0.25 })).toBe(false);
|
|
68
|
+
});
|
|
69
|
+
it("treats unknown core count (0) as not-CPU-busy so we never divide by zero", () => {
|
|
70
|
+
expect(machineAlreadyBusy({ cores: 0, load1: 99, availableMemFraction: 0.6 })).toBe(false);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
describe("resolveUploadByteBudget", () => {
|
|
74
|
+
const KEY = "HQ_SYNC_UPLOAD_BYTE_BUDGET_MB";
|
|
75
|
+
let saved;
|
|
76
|
+
const MiB = 1024 * 1024;
|
|
77
|
+
const GiB = 1024 * MiB;
|
|
78
|
+
beforeEach(() => {
|
|
79
|
+
saved = process.env[KEY];
|
|
80
|
+
delete process.env[KEY];
|
|
81
|
+
});
|
|
82
|
+
afterEach(() => {
|
|
83
|
+
if (saved === undefined)
|
|
84
|
+
delete process.env[KEY];
|
|
85
|
+
else
|
|
86
|
+
process.env[KEY] = saved;
|
|
87
|
+
});
|
|
88
|
+
it("sizes to 25% of AVAILABLE memory", () => {
|
|
89
|
+
// 8 GiB total, 20% available => 1.6 GiB available, 25% of that = 400 MiB.
|
|
90
|
+
expect(resolveUploadByteBudget(0.2, 8 * GiB)).toBe(Math.floor(0.2 * 8 * GiB * 0.25));
|
|
91
|
+
});
|
|
92
|
+
it("caps at 1 GiB when available memory is plentiful", () => {
|
|
93
|
+
expect(resolveUploadByteBudget(0.9, 64 * GiB)).toBe(GiB);
|
|
94
|
+
});
|
|
95
|
+
it("shrinks with available memory on a tight host (no oversized floor)", () => {
|
|
96
|
+
// 8 GiB total, 500 MiB available => 25% = 125 MiB. A total-RAM budget would
|
|
97
|
+
// have allowed ~819 MiB here, more than what's free.
|
|
98
|
+
const availFrac = (500 * MiB) / (8 * GiB);
|
|
99
|
+
expect(resolveUploadByteBudget(availFrac, 8 * GiB)).toBe(Math.floor(500 * MiB * 0.25));
|
|
100
|
+
});
|
|
101
|
+
it("never returns below 1 byte even when available rounds to ~0", () => {
|
|
102
|
+
expect(resolveUploadByteBudget(0, 8 * GiB)).toBe(1);
|
|
103
|
+
});
|
|
104
|
+
it("honors an explicit fractional-MB override regardless of available memory", () => {
|
|
105
|
+
process.env[KEY] = "0.005"; // ~5 KiB, for driving the pool test
|
|
106
|
+
expect(resolveUploadByteBudget(0.9, 64 * GiB)).toBe(Math.floor(0.005 * MiB));
|
|
107
|
+
});
|
|
108
|
+
it("ignores a malformed override and uses the available-based default", () => {
|
|
109
|
+
process.env[KEY] = "nope";
|
|
110
|
+
expect(resolveUploadByteBudget(0.2, 8 * GiB)).toBe(Math.floor(0.2 * 8 * GiB * 0.25));
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
//# sourceMappingURL=sync-core.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync-core.test.js","sourceRoot":"","sources":["../src/sync-core.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,uBAAuB,GAExB,MAAM,gBAAgB,CAAC;AAExB,gFAAgF;AAChF,MAAM,IAAI,GAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC;AAChF,6EAA6E;AAC7E,MAAM,QAAQ,GAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC;AACpF,gEAAgE;AAChE,MAAM,SAAS,GAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC;AAErF,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,MAAM,GAAG,GAAG,8BAA8B,CAAC;IAC3C,IAAI,KAAyB,CAAC;IAC9B,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;;YAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,0BAA0B,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,MAAM,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,MAAM,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;QACxB,MAAM,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,GAAG,EAAE;QACxF,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,0BAA0B,EAAE,CAAC;QACvC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,oCAAoC;QACpC,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,uCAAuC;QACvC,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,MAAM,GAAG,GAAG,+BAA+B,CAAC;IAC5C,IAAI,KAAyB,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,GAAG,GAAG,CAAC;IACvB,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;;YAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,0EAA0E;QAC1E,MAAM,CAAC,uBAAuB,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,CAAC,uBAAuB,CAAC,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,4EAA4E;QAC5E,qDAAqD;QACrD,MAAM,SAAS,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1C,MAAM,CAAC,uBAAuB,CAAC,SAAS,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,oCAAoC;QAChE,MAAM,CAAC,uBAAuB,CAAC,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QAC1B,MAAM,CAAC,uBAAuB,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|