@indigoai-us/hq-cloud 6.11.11 → 6.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +5 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +76 -978
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +75 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +320 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +276 -660
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +541 -748
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +382 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +149 -30
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +8 -3
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +85 -16
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +213 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +12 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -17
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +67 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +38 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +155 -143
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +99 -1054
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +81 -0
- package/src/cli/rescue-core.ts +400 -165
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +420 -693
- package/src/cli/sync.test.ts +460 -1
- package/src/cli/sync.ts +788 -825
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +183 -31
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +235 -1
- package/src/remote-pull.ts +106 -18
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +73 -1
- package/src/sync/push-receiver.ts +56 -20
- package/src/sync-core.ts +58 -0
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +215 -174
package/dist/cli/share.js
CHANGED
|
@@ -15,7 +15,8 @@ import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
|
|
|
15
15
|
import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
16
16
|
import { resolveConflict } from "./conflict.js";
|
|
17
17
|
import { fetchCompanyTombstones, } from "./tombstones.js";
|
|
18
|
-
import { isCoveredByAny, isDirInScope } from "../prefix-coalesce.js";
|
|
18
|
+
import { isCoveredByAny, isDirInScope, } from "../prefix-coalesce.js";
|
|
19
|
+
import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, } from "../sync-core.js";
|
|
19
20
|
import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
20
21
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
21
22
|
/**
|
|
@@ -334,20 +335,6 @@ function computePushPlan(filesToShare, journal, skipUnchanged) {
|
|
|
334
335
|
}
|
|
335
336
|
return { items, filesToUpload, bytesToUpload, filesToSkip };
|
|
336
337
|
}
|
|
337
|
-
/**
|
|
338
|
-
* Is this error the S3/STS "access denied" class? Expected when a scoped
|
|
339
|
-
* member/guest credential touches a key outside its granted ACL prefixes
|
|
340
|
-
* (the server's `SCOPE_EXCEEDS_PARENT` surfaces as a 403 AccessDenied /
|
|
341
|
-
* Forbidden). Mirrors the pull-side `isAccessDenied` in sync.ts so the push
|
|
342
|
-
* leg can treat a stray out-of-scope key as a skip rather than a fatal throw.
|
|
343
|
-
*/
|
|
344
|
-
function isAccessDenied(err) {
|
|
345
|
-
if (err && typeof err === "object" && "name" in err) {
|
|
346
|
-
const name = err.name;
|
|
347
|
-
return name === "AccessDenied" || name === "Forbidden";
|
|
348
|
-
}
|
|
349
|
-
return false;
|
|
350
|
-
}
|
|
351
338
|
/**
|
|
352
339
|
* A conditional-write fence rejection — the SDK's 412 (`name:
|
|
353
340
|
* "PreconditionFailed"`) or the presigned transport's mirror of it. Means
|
|
@@ -398,21 +385,25 @@ function wrapFilterWithScope(underlying, syncRoot, prefixSet, onScopeExcluded) {
|
|
|
398
385
|
* Share local file(s) to the entity vault.
|
|
399
386
|
*/
|
|
400
387
|
export async function share(options) {
|
|
388
|
+
const run = await createPushRunContext(options);
|
|
389
|
+
const counters = createShareCounters();
|
|
390
|
+
const filesRefusedStalePaths = [];
|
|
391
|
+
const conflictPaths = [];
|
|
392
|
+
const plans = await buildSharePlans(run);
|
|
393
|
+
const uploadResult = await executeUploads(run, plans.pushPlan, counters, conflictPaths);
|
|
394
|
+
if (uploadResult.aborted) {
|
|
395
|
+
return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, true);
|
|
396
|
+
}
|
|
397
|
+
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths);
|
|
398
|
+
finalizeShareJournal(run);
|
|
399
|
+
throwUploadWorkerErrors(uploadResult.workerErrors);
|
|
400
|
+
return buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths, false);
|
|
401
|
+
}
|
|
402
|
+
const REFUSED_STALE_PATH_CAP = 50;
|
|
403
|
+
async function createPushRunContext(options) {
|
|
401
404
|
const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged, propagateDeletes } = options;
|
|
402
|
-
// Default to "owned-only" — the pre-5.24 behavior — when delete-propagation
|
|
403
|
-
// is on but the caller hasn't pinned a policy. Staged-default rollout
|
|
404
|
-
// (see CHANGELOG / PR for hq-cloud 5.24.0): 5.24 ships the currency-gated
|
|
405
|
-
// CODE PATH plus the conflict-mirror exclusion (which is policy-
|
|
406
|
-
// independent and immediately stops new litter), but holds the default
|
|
407
|
-
// flip to a later release after soak. Opt into the safer policy now via
|
|
408
|
-
// `propagateDeletePolicy: "currency-gated"` (explicit) or
|
|
409
|
-
// `HQ_SYNC_DELETE_POLICY=currency-gated` (env, honored by sync-runner).
|
|
410
|
-
// The default flip to `"currency-gated"` is scheduled for 5.25.0.
|
|
411
405
|
let propagateDeletePolicy = options.propagateDeletePolicy ?? "owned-only";
|
|
412
406
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
413
|
-
// Exactly-one-of contract: either we vend (vaultConfig) or the caller did
|
|
414
|
-
// (entityContext). Both supplied is ambiguous (which credentials win?), and
|
|
415
|
-
// neither leaves us with no way to talk to S3.
|
|
416
407
|
if (vaultConfig && entityContext) {
|
|
417
408
|
throw new Error("share() requires exactly one of `vaultConfig` or `entityContext`, not both. " +
|
|
418
409
|
"Pass `vaultConfig` to vend credentials internally, or `entityContext` to use pre-vended ones.");
|
|
@@ -421,72 +412,20 @@ export async function share(options) {
|
|
|
421
412
|
throw new Error("share() requires either `vaultConfig` (for internal STS vending) " +
|
|
422
413
|
"or `entityContext` (pre-vended credentials).");
|
|
423
414
|
}
|
|
424
|
-
// Resolve company — slug, UID, or from active config. When the caller
|
|
425
|
-
// provided a pre-resolved entityContext, prefer its slug as the canonical
|
|
426
|
-
// ref (the caller already knows what entity these creds are for).
|
|
427
415
|
const companyRef = company ?? entityContext?.slug ?? resolveActiveCompany(hqRoot);
|
|
428
416
|
if (!companyRef) {
|
|
429
417
|
throw new Error("No company specified and no active company found. " +
|
|
430
418
|
"Use --company <slug> or set up .hq/config.json.");
|
|
431
419
|
}
|
|
432
|
-
|
|
433
|
-
// 1. vaultConfig provided → resolveEntityContext does the lookup + STS vend
|
|
434
|
-
// (cached + auto-refreshable mid-run).
|
|
435
|
-
// 2. entityContext provided → use it directly. No lookup, no vending,
|
|
436
|
-
// no auto-refresh (we have no Cognito token to re-vend with).
|
|
437
|
-
// Caller is responsible for vending credentials with enough TTL to
|
|
438
|
-
// cover the run; if they under-vend, the AWS SDK surfaces ExpiredToken
|
|
439
|
-
// naturally on the first failing PUT.
|
|
440
|
-
let ctx = entityContext
|
|
420
|
+
const ctx = entityContext
|
|
441
421
|
? entityContext
|
|
442
422
|
: await resolveEntityContext(companyRef, vaultConfig);
|
|
443
|
-
// Personal-vault policy correction (6.0.1). The `owned-only` rule encodes a
|
|
444
|
-
// multi-user curation premise — "don't tombstone peer-uploaded content even
|
|
445
|
-
// if my journal says I pulled it" — which is meaningful when several humans
|
|
446
|
-
// share a company bucket (a behind machine's first sync must not erase
|
|
447
|
-
// recent uploads from peers). On a personal vault that premise collapses:
|
|
448
|
-
// every file is the same human's content, just routed through different
|
|
449
|
-
// machines, and `direction: "down"` only means "uploaded from my laptop,
|
|
450
|
-
// pulled by my EC2" — it never means "uploaded by someone else." With
|
|
451
|
-
// `owned-only` in effect, `rm <file>` followed by `hq sync` silently fails
|
|
452
|
-
// to propagate the delete, leaving permanent vault litter (the May-27
|
|
453
|
-
// `personal/.obsidian/*.drift-*` files were diagnosed exactly this way).
|
|
454
|
-
// The etag-based `currency-gated` policy already captures the only safety
|
|
455
|
-
// intent that survives the single-user case ("don't tombstone if remote
|
|
456
|
-
// drifted since I last synced"); coerce to it here so the policy is right
|
|
457
|
-
// regardless of which caller's default landed. Explicit `"all"` is
|
|
458
|
-
// preserved — it's the emergency-reconcile opt-out and the caller has
|
|
459
|
-
// already asserted intent.
|
|
460
423
|
if (ctx.uid.startsWith("prs_") && propagateDeletePolicy === "owned-only") {
|
|
461
424
|
propagateDeletePolicy = "currency-gated";
|
|
462
425
|
}
|
|
463
|
-
// Remote keys are company-relative; the on-disk scoping prefix is
|
|
464
|
-
// companies/{slug}/. Anything outside this folder gets skipped to avoid
|
|
465
|
-
// leaking cross-company state into the vault.
|
|
466
|
-
//
|
|
467
|
-
// In personalMode the syncRoot is `hqRoot` itself — remote keys are
|
|
468
|
-
// hq-root-relative to match the Rust personal first-push (which uploads
|
|
469
|
-
// every non-excluded top-level dir under ~/HQ). The exclusion list is
|
|
470
|
-
// enforced upstream by the runner; share() just trusts `paths`.
|
|
471
426
|
const syncRoot = options.personalMode === true
|
|
472
427
|
? hqRoot
|
|
473
428
|
: path.join(hqRoot, "companies", ctx.slug);
|
|
474
|
-
// Personal-vault default exclusions (introduced in 5.25): wrap the base
|
|
475
|
-
// ignore filter so paths matching `PERSONAL_VAULT_DEFAULT_EXCLUSIONS` are
|
|
476
|
-
// rejected before they upload OR enter the delete plan. Refuses & warns —
|
|
477
|
-
// an already-leaked remote object stays put as an orphan; a separate one-
|
|
478
|
-
// shot purge handles legacy litter.
|
|
479
|
-
//
|
|
480
|
-
// Out-of-policy hits are deduplicated in `excludedSet` so the same path
|
|
481
|
-
// hitting the filter from both the upload walk and the delete-plan walk
|
|
482
|
-
// counts once. `excludedById` powers the per-rule breakdown on the
|
|
483
|
-
// `personal-vault-out-of-policy` event so UI can render which class
|
|
484
|
-
// (secret / machine-local / scratch / …) did the work.
|
|
485
|
-
//
|
|
486
|
-
// Company-mode syncs skip this wrap entirely — company vaults have their
|
|
487
|
-
// own first-push protection (settings/, data/, workers/, .git/) defined
|
|
488
|
-
// in hq-sync's Rust util/ignore.rs, and a company may legitimately ship
|
|
489
|
-
// `output/` or `.env*` paths inside its `companies/{slug}/data/` folder.
|
|
490
429
|
const ignoreFilter = createIgnoreFilter(hqRoot);
|
|
491
430
|
const excludedSet = new Set();
|
|
492
431
|
const excludedById = {};
|
|
@@ -496,12 +435,6 @@ export async function share(options) {
|
|
|
496
435
|
excludedSet.add(rel);
|
|
497
436
|
excludedById[match.id] = (excludedById[match.id] ?? 0) + 1;
|
|
498
437
|
};
|
|
499
|
-
// ACL scope filter (member/guest scoped push). The vended child credential
|
|
500
|
-
// is scoped to `options.prefixSet`; any candidate outside those prefixes
|
|
501
|
-
// would draw the server's correct 403 SCOPE_EXCEEDS_PARENT on PUT and abort
|
|
502
|
-
// the whole company. Pre-filter the plan to the granted subset instead —
|
|
503
|
-
// the push-side analogue of the pull leg's `skip-out-of-scope` (US-005).
|
|
504
|
-
// `undefined` = owner/`all` → no scope filter (full access).
|
|
505
438
|
const scopeExcludedSet = new Set();
|
|
506
439
|
const onScopeExcluded = (rel) => {
|
|
507
440
|
scopeExcludedSet.add(rel);
|
|
@@ -513,106 +446,67 @@ export async function share(options) {
|
|
|
513
446
|
? wrapFilterWithScope(baseFilter, syncRoot, options.prefixSet, onScopeExcluded)
|
|
514
447
|
: baseFilter;
|
|
515
448
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
516
|
-
// Seed the canonical personal-vault journal from the legacy `personal` file
|
|
517
|
-
// exactly once — engine-side so every consumer (sync-runner, hq-cli) gets
|
|
518
|
-
// it; see the matching guard in sync.ts.
|
|
519
449
|
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
|
|
520
450
|
migratePersonalVaultJournal();
|
|
521
451
|
const journal = readJournal(journalSlug);
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
452
|
+
return {
|
|
453
|
+
options,
|
|
454
|
+
paths,
|
|
455
|
+
message,
|
|
456
|
+
onConflict,
|
|
457
|
+
vaultConfig,
|
|
458
|
+
entityContext,
|
|
459
|
+
hqRoot,
|
|
460
|
+
skipUnchanged,
|
|
461
|
+
propagateDeletes,
|
|
462
|
+
propagateDeletePolicy,
|
|
463
|
+
emit,
|
|
464
|
+
companyRef,
|
|
465
|
+
ctx,
|
|
466
|
+
syncRoot,
|
|
467
|
+
shouldSync,
|
|
468
|
+
journalSlug,
|
|
469
|
+
journal,
|
|
470
|
+
excludedSet,
|
|
471
|
+
excludedById,
|
|
472
|
+
scopeExcludedSet,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
function createShareCounters() {
|
|
476
|
+
return {
|
|
477
|
+
filesUploaded: 0,
|
|
478
|
+
bytesUploaded: 0,
|
|
479
|
+
filesSkipped: 0,
|
|
480
|
+
filesDeleted: 0,
|
|
481
|
+
filesTombstoned: 0,
|
|
482
|
+
filesRefusedStale: 0,
|
|
483
|
+
filesSuppressedByTombstone: 0,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
async function buildSharePlans(run) {
|
|
487
|
+
const filesToShare = collectFiles(run.paths, run.hqRoot, run.syncRoot, run.shouldSync);
|
|
488
|
+
const pushPlan = computePushPlan(filesToShare, run.journal, run.skipUnchanged === true);
|
|
489
|
+
const deleteScopeRoots = run.propagateDeletes === true
|
|
490
|
+
? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot)
|
|
555
491
|
: [];
|
|
556
|
-
const deletePlan = propagateDeletes === true
|
|
557
|
-
? await computeDeletePlan(journal, syncRoot, deleteScopeRoots, shouldSync, propagateDeletePolicy, ctx)
|
|
492
|
+
const deletePlan = run.propagateDeletes === true
|
|
493
|
+
? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx)
|
|
558
494
|
: { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
// company's `companies/{slug}/` keys in the personal bucket). Independent
|
|
562
|
-
// of `propagateDeletes` and DOES NOT require the local file to be missing
|
|
563
|
-
// — the caller is making a stronger claim than "local says delete this".
|
|
564
|
-
// Honors the same owned-only safety policy so a misconfigured caller
|
|
565
|
-
// can never erase content the journal records as pulled from elsewhere.
|
|
566
|
-
// Dedupes against `deletePlan` so a key in both plans is only processed
|
|
567
|
-
// once (DeleteObject is idempotent on S3 but the journal-write would
|
|
568
|
-
// race a no-op pass through the loop body).
|
|
569
|
-
const decommissionPlan = (options.decommissionPrefixes ?? []).length > 0
|
|
570
|
-
? computeDecommissionPlan(journal, options.decommissionPrefixes ?? [], propagateDeletePolicy,
|
|
571
|
-
// Dedup against `toDelete` (decommission and propagate-delete
|
|
572
|
-
// would both issue DeleteObject — single call wins) and against
|
|
573
|
-
// `toTombstone` (the remote is already 404; the tombstone loop
|
|
574
|
-
// drops the journal entry without a network call — decommission
|
|
575
|
-
// yields, both for efficiency and to avoid emitting two
|
|
576
|
-
// "deleted" events for the same key).
|
|
577
|
-
//
|
|
578
|
-
// We do NOT dedup against `refusedStale`. A key whose remote
|
|
579
|
-
// ETag drifted (peer wrote a newer version) but which decommission
|
|
580
|
-
// claims should still be removed — the caller has asserted this
|
|
581
|
-
// key doesn't belong in this bucket regardless of peer activity.
|
|
582
|
-
// Under owned-only (default) `computeDecommissionPlan`'s
|
|
583
|
-
// direction:'up' filter already excludes peer-written entries;
|
|
584
|
-
// under policy:'all' the caller has opted out of that safety
|
|
585
|
-
// anyway. The refusedStale loop below filters out keys we're
|
|
586
|
-
// about to decommission to avoid emitting a spurious "kept on
|
|
587
|
-
// remote" event for content we're deleting.
|
|
588
|
-
new Set([...deletePlan.toDelete, ...deletePlan.toTombstone]))
|
|
495
|
+
const decommissionPlan = (run.options.decommissionPrefixes ?? []).length > 0
|
|
496
|
+
? computeDecommissionPlan(run.journal, run.options.decommissionPrefixes ?? [], run.propagateDeletePolicy, new Set([...deletePlan.toDelete, ...deletePlan.toTombstone]))
|
|
589
497
|
: [];
|
|
590
|
-
emit({
|
|
498
|
+
run.emit({
|
|
591
499
|
type: "plan",
|
|
592
|
-
// share() is push-only; pull counts are sourced from sync()'s plan event.
|
|
593
500
|
filesToDownload: 0,
|
|
594
501
|
bytesToDownload: 0,
|
|
595
|
-
filesToUpload:
|
|
596
|
-
bytesToUpload:
|
|
597
|
-
filesToSkip:
|
|
598
|
-
// Push conflicts require a remote HEAD; we don't yet do that in Stage 1,
|
|
599
|
-
// so this stays 0. V1.5 (single LIST) will let us classify them up-front.
|
|
502
|
+
filesToUpload: pushPlan.filesToUpload,
|
|
503
|
+
bytesToUpload: pushPlan.bytesToUpload,
|
|
504
|
+
filesToSkip: pushPlan.filesToSkip,
|
|
600
505
|
filesToConflict: 0,
|
|
601
|
-
// Reported count is the deletes we're actually going to issue — does NOT
|
|
602
|
-
// include tombstones (no S3 call) or refused-stale (no journal change).
|
|
603
|
-
// Refusals surface as their own event stream so consumers that care can
|
|
604
|
-
// render a "kept on remote: N" line separately. `decommissionPlan` adds
|
|
605
|
-
// to this count because every decommission entry IS an issued
|
|
606
|
-
// DeleteObject (different intent than propagate-deletes, same network
|
|
607
|
-
// effect).
|
|
608
506
|
filesToDelete: deletePlan.toDelete.length + decommissionPlan.length,
|
|
609
507
|
});
|
|
610
|
-
// Bulk-asymmetry summary event. Emitted once if the circuit-breaker
|
|
611
|
-
// tripped inside `computeDeletePlan` — see DeletePlan.bulkAsymmetry doc.
|
|
612
|
-
// Per-key refusal events fire later in the refusedStale loop with
|
|
613
|
-
// reason: "bulk-asymmetry" so the UI can also show the affected paths.
|
|
614
508
|
if (deletePlan.bulkAsymmetry) {
|
|
615
|
-
emit({
|
|
509
|
+
run.emit({
|
|
616
510
|
type: "delete-refused-bulk-asymmetry",
|
|
617
511
|
candidates: deletePlan.bulkAsymmetry.candidates,
|
|
618
512
|
inScope: deletePlan.bulkAsymmetry.inScope,
|
|
@@ -620,197 +514,85 @@ export async function share(options) {
|
|
|
620
514
|
samplePaths: deletePlan.bulkAsymmetry.samplePaths,
|
|
621
515
|
});
|
|
622
516
|
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
// (`TRANSFER_CONCURRENCY`, default 16, tunable via
|
|
628
|
-
// `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
|
|
629
|
-
// syncs. Skip-size-limit and skip-unchanged are handled inline first
|
|
630
|
-
// (pure local-state mutation). Upload candidates are collected into
|
|
631
|
-
// `uploadItems[]` and processed in parallel via the pool below.
|
|
632
|
-
//
|
|
633
|
-
// Abort handling: when any item's conflict resolution is "abort", we
|
|
634
|
-
// set `aborted = true` so the pool stops queueing new items, drain
|
|
635
|
-
// in-flight cleanly, and short-circuit to the abort return. In-flight
|
|
636
|
-
// PUTs that already issued will complete (S3 doesn't have client-side
|
|
637
|
-
// cancellation in this code path); their results are still recorded on
|
|
638
|
-
// the journal so the next sync's planner doesn't re-fire them.
|
|
639
|
-
// Interactive-mode prompts: when `onConflict` is unset the per-item conflict
|
|
640
|
-
// path calls resolveConflict()'s readline prompt on process.stdin, and two
|
|
641
|
-
// pool workers prompting at once would race for the terminal and interleave
|
|
642
|
-
// answers. The 5.36.x guard solved this by forcing the WHOLE pool to
|
|
643
|
-
// concurrency=1 — which made an interactive `hq sync now` crawl even when
|
|
644
|
-
// zero conflicts existed (every transfer serialized just in case one might
|
|
645
|
-
// prompt). Instead, keep full env-tunable concurrency and serialize ONLY the
|
|
646
|
-
// prompt (see `resolveConflictSerialized` below): at most one prompt awaits
|
|
647
|
-
// input at a time while transfers stay parallel.
|
|
648
|
-
const TRANSFER_CONCURRENCY = (() => {
|
|
649
|
-
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
650
|
-
if (raw === undefined || raw === "")
|
|
651
|
-
return 16;
|
|
652
|
-
const parsed = Number.parseInt(raw, 10);
|
|
653
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
654
|
-
})();
|
|
655
|
-
// Chained lock around the (possibly interactive) conflict prompt. Each
|
|
656
|
-
// resolveConflict() runs only after the previous one settles, so concurrent
|
|
657
|
-
// pool workers never prompt over each other on stdin — without dropping the
|
|
658
|
-
// transfer pool's parallelism. A rejected prompt must not wedge the chain,
|
|
659
|
-
// so the link swallows errors (the original promise still rejects to its
|
|
660
|
-
// awaiter). In non-interactive mode resolveConflict applies the configured
|
|
661
|
-
// strategy without reading stdin, so the lock adds no real serialization.
|
|
517
|
+
return { pushPlan, deletePlan, decommissionPlan };
|
|
518
|
+
}
|
|
519
|
+
async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
520
|
+
const TRANSFER_CONCURRENCY = resolveTransferConcurrency();
|
|
662
521
|
let conflictPromptChain = Promise.resolve();
|
|
663
522
|
const resolveConflictSerialized = (info) => {
|
|
664
|
-
const
|
|
665
|
-
conflictPromptChain =
|
|
666
|
-
return
|
|
523
|
+
const resolution = conflictPromptChain.then(() => resolveConflict(info, run.onConflict));
|
|
524
|
+
conflictPromptChain = resolution.then(() => undefined, () => undefined);
|
|
525
|
+
return resolution;
|
|
667
526
|
};
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
// object; the pull side already honors it. But the PUSH side did not: a behind
|
|
672
|
-
// peer who still holds the deleted file locally would re-upload it here, and
|
|
673
|
-
// because the re-uploaded object post-dates the tombstone, the pull planner's
|
|
674
|
-
// timestamp-only re-create heuristic (`isRemoteRecreateAfterTombstone`) treats
|
|
675
|
-
// it as a genuine re-create and resurrects the key for EVERYONE — defeating the
|
|
676
|
-
// authoritative delete. Consult the tombstones so a stale-baseline upload is
|
|
677
|
-
// skipped at the source.
|
|
678
|
-
//
|
|
679
|
-
// Source: an injected `fileTombstones` (a sync run can hand the push leg the
|
|
680
|
-
// map it already fetched for the pull leg), else a self-fetch for COMPANY
|
|
681
|
-
// vaults that have a `vaultConfig`. Personal vaults have no company tombstones
|
|
682
|
-
// (the pull side skips them too), and `entityContext`-only callers have no
|
|
683
|
-
// auth to fetch with — both degrade to an empty map (no suppression), the
|
|
684
|
-
// safe/legacy direction.
|
|
685
|
-
const fileTombstones = options.fileTombstones ??
|
|
686
|
-
(!ctx.uid.startsWith("prs_") && vaultConfig
|
|
687
|
-
? await fetchCompanyTombstones(vaultConfig, ctx.uid)
|
|
527
|
+
const fileTombstones = run.options.fileTombstones ??
|
|
528
|
+
(!run.ctx.uid.startsWith("prs_") && run.vaultConfig
|
|
529
|
+
? await fetchCompanyTombstones(run.vaultConfig, run.ctx.uid)
|
|
688
530
|
: new Map());
|
|
689
|
-
// Phase A: serial classification pass — handle skips inline, collect
|
|
690
|
-
// upload candidates for the parallel pool.
|
|
691
531
|
const uploadItems = [];
|
|
692
|
-
for (const item of
|
|
532
|
+
for (const item of pushPlan.items) {
|
|
693
533
|
if (item.action === "skip-size-limit") {
|
|
694
|
-
emit({
|
|
534
|
+
run.emit({
|
|
695
535
|
type: "error",
|
|
696
536
|
path: item.relativePath,
|
|
697
537
|
message: "file exceeds size limit",
|
|
698
538
|
});
|
|
699
|
-
filesSkipped++;
|
|
539
|
+
counters.filesSkipped++;
|
|
700
540
|
continue;
|
|
701
541
|
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
filesSuppressedByTombstone++;
|
|
717
|
-
emit({
|
|
718
|
-
type: "upload-suppressed-tombstone",
|
|
719
|
-
path: item.relativePath,
|
|
720
|
-
deletedAt: ts.deletedAt,
|
|
721
|
-
});
|
|
722
|
-
continue;
|
|
542
|
+
if (item.action === "upload") {
|
|
543
|
+
if (fileTombstones.size > 0) {
|
|
544
|
+
const ts = fileTombstones.get(toPosixKey(item.relativePath));
|
|
545
|
+
if (ts !== undefined) {
|
|
546
|
+
const entry = run.journal.files[item.relativePath];
|
|
547
|
+
if (entry && entry.hash === item.localHash) {
|
|
548
|
+
counters.filesSuppressedByTombstone++;
|
|
549
|
+
run.emit({
|
|
550
|
+
type: "upload-suppressed-tombstone",
|
|
551
|
+
path: item.relativePath,
|
|
552
|
+
deletedAt: ts.deletedAt,
|
|
553
|
+
});
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
723
556
|
}
|
|
724
557
|
}
|
|
558
|
+
uploadItems.push(item);
|
|
559
|
+
continue;
|
|
725
560
|
}
|
|
726
|
-
if (item.
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
// never alters the content hash, so this stays a pure no-op skip. The
|
|
732
|
-
// journal is persisted unconditionally by writeJournal at the end of the
|
|
733
|
-
// run, so this survives even when nothing uploads.
|
|
734
|
-
if (item.restamp) {
|
|
735
|
-
const existing = journal.files[item.relativePath];
|
|
736
|
-
if (existing && existing.hash) {
|
|
737
|
-
existing.mtimeMs = item.restamp.mtimeMs;
|
|
738
|
-
existing.size = item.restamp.size;
|
|
739
|
-
}
|
|
561
|
+
if (item.restamp) {
|
|
562
|
+
const existing = run.journal.files[item.relativePath];
|
|
563
|
+
if (existing && existing.hash) {
|
|
564
|
+
existing.mtimeMs = item.restamp.mtimeMs;
|
|
565
|
+
existing.size = item.restamp.size;
|
|
740
566
|
}
|
|
741
|
-
filesSkipped++;
|
|
742
|
-
continue;
|
|
743
567
|
}
|
|
744
|
-
|
|
568
|
+
counters.filesSkipped++;
|
|
745
569
|
}
|
|
746
|
-
|
|
747
|
-
// signing the SAME metadata the pool below computes so each task replays the
|
|
748
|
-
// cached headers and skips its own presign. Turns an N-file push from ~N
|
|
749
|
-
// presign calls into ceil(N/1000) GET + ceil(N/1000) PUT — keeping a bulk
|
|
750
|
-
// push under the 100/hr limit. No-op on the S3 SDK transport; best-effort.
|
|
751
|
-
await primeUploads(ctx, uploadItems.map((it) => ({
|
|
570
|
+
await primeUploads(run.ctx, uploadItems.map((it) => ({
|
|
752
571
|
key: it.relativePath,
|
|
753
572
|
localPath: it.absolutePath,
|
|
754
573
|
isSymlink: it.kind === "symlink",
|
|
755
|
-
author: options.author,
|
|
574
|
+
author: run.options.author,
|
|
756
575
|
})));
|
|
757
|
-
//
|
|
758
|
-
//
|
|
759
|
-
//
|
|
760
|
-
|
|
576
|
+
// Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
|
|
577
|
+
// large upload set doesn't mint one presign per HEAD and burst/trip the
|
|
578
|
+
// presign breaker. Mirrors the new-files + tombstone pre-primes on the pull.
|
|
579
|
+
await primeObjectTransport(run.ctx, "get", uploadItems.map((it) => it.relativePath));
|
|
761
580
|
let aborted = false;
|
|
762
581
|
let abortFlightConflictPaths = [];
|
|
763
582
|
const processUploadItem = async (item) => {
|
|
764
583
|
if (aborted)
|
|
765
584
|
return;
|
|
766
585
|
const { absolutePath, relativePath, localHash } = item;
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
// from, so we let the AWS SDK surface ExpiredToken naturally on the
|
|
770
|
-
// PUT below if the caller under-vended.
|
|
771
|
-
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
772
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
586
|
+
if (run.vaultConfig && isExpiringSoon(run.ctx.expiresAt)) {
|
|
587
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
773
588
|
}
|
|
774
|
-
// Check for remote conflict — refuse to overwrite newer remote version.
|
|
775
|
-
//
|
|
776
|
-
// A real conflict requires BOTH sides to have moved since the last sync.
|
|
777
|
-
// The previous predicate only checked `journalEntry.hash !== localHash`,
|
|
778
|
-
// which mislabelled every local edit as a conflict and (combined with
|
|
779
|
-
// `--on-conflict keep`) silently dropped the user's edit. We now compare
|
|
780
|
-
// the current remote ETag against the one captured at last sync; when
|
|
781
|
-
// missing (legacy entries), we fall back to the same `lastModified >
|
|
782
|
-
// syncedAt` heuristic the pull side uses.
|
|
783
|
-
//
|
|
784
|
-
// Bug #7 (data-loss class — see workspace/reports/hq-cloud-5.33.0-
|
|
785
|
-
// deep-test.md): for a path with NO prior journal entry (first push
|
|
786
|
-
// from this machine), the localChanged/remoteChanged predicates above
|
|
787
|
-
// both evaluate FALSE (their guards require `!!journalEntry`). Push
|
|
788
|
-
// fell through to an unconditional PUT, silently clobbering any
|
|
789
|
-
// peer's content already at that key. The verification report's V7
|
|
790
|
-
// isolated this — the bug is independent of \`--on-conflict\` mode;
|
|
791
|
-
// it's keyed on "do I have a prior journal entry?" not on the flag.
|
|
792
|
-
//
|
|
793
|
-
// Fresh-collision branch: when remoteMeta exists and there's no
|
|
794
|
-
// journal entry, hash the local body (MD5 for parity with S3's
|
|
795
|
-
// single-part etag) and compare. Match → no conflict, silently skip
|
|
796
|
-
// the PUT (the bytes are already there). Mismatch → treat as a
|
|
797
|
-
// conflict in the same shared branch below.
|
|
798
|
-
// Defense-in-depth for the scoped-push 403: the `prefixSet` filter above
|
|
799
|
-
// should already have dropped any out-of-scope key from the plan, but a
|
|
800
|
-
// grant that changed mid-run, a pinned prefix outside the grant, or
|
|
801
|
-
// prefix-coalesce imprecision can still leave an out-of-scope key here.
|
|
802
|
-
// This HEAD sits OUTSIDE the per-file PUT try/catch below, so a thrown
|
|
803
|
-
// 403 used to bubble to `workerErrors` and abort the ENTIRE company with
|
|
804
|
-
// a generic message and exit 2. Catch the access-denied class, surface
|
|
805
|
-
// the offending PATH clearly, and skip just this key — the rest of the
|
|
806
|
-
// company still syncs. Non-access-denied errors re-throw unchanged.
|
|
807
589
|
let remoteMeta;
|
|
808
590
|
try {
|
|
809
|
-
remoteMeta = await headRemoteFile(ctx, relativePath);
|
|
591
|
+
remoteMeta = await headRemoteFile(run.ctx, relativePath);
|
|
810
592
|
}
|
|
811
593
|
catch (headErr) {
|
|
812
594
|
if (isAccessDenied(headErr)) {
|
|
813
|
-
emit({
|
|
595
|
+
run.emit({
|
|
814
596
|
type: "error",
|
|
815
597
|
path: relativePath,
|
|
816
598
|
message: "skipped: outside granted ACL scope (server returned 403 " +
|
|
@@ -822,18 +604,15 @@ export async function share(options) {
|
|
|
822
604
|
throw headErr;
|
|
823
605
|
}
|
|
824
606
|
if (remoteMeta) {
|
|
825
|
-
const journalEntry = journal.files[relativePath];
|
|
607
|
+
const journalEntry = run.journal.files[relativePath];
|
|
826
608
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
827
609
|
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
|
|
828
610
|
let isFreshCollision = false;
|
|
829
611
|
let multipartConverged = false;
|
|
830
|
-
if (!journalEntry && item.kind === "
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
// prefix + target) isn't a pure byte mirror and would mis-
|
|
835
|
-
// classify; symlink overwrites are rare and an audit pass after
|
|
836
|
-
// the broader bug-cleanup wave can extend coverage if needed.
|
|
612
|
+
if (!journalEntry && item.kind === "symlink") {
|
|
613
|
+
isFreshCollision = true;
|
|
614
|
+
}
|
|
615
|
+
else if (!journalEntry && item.kind === "file") {
|
|
837
616
|
const remoteEtagNormalized = normalizeEtag(remoteMeta.etag);
|
|
838
617
|
const isMultipart = /-\d+$/.test(remoteEtagNormalized);
|
|
839
618
|
if (!isMultipart) {
|
|
@@ -842,44 +621,22 @@ export async function share(options) {
|
|
|
842
621
|
if (localMd5 !== remoteEtagNormalized) {
|
|
843
622
|
isFreshCollision = true;
|
|
844
623
|
}
|
|
845
|
-
// Match → bytes are already there; fall through to upload
|
|
846
|
-
// path which is idempotent (S3 will overwrite with identical
|
|
847
|
-
// content + carry our metadata). Cheap, no behavior change.
|
|
848
624
|
}
|
|
849
625
|
else {
|
|
850
|
-
|
|
851
|
-
// content hash, so — unlike the single-part branch — we cannot
|
|
852
|
-
// decide collision-vs-identical from the etag alone. The old
|
|
853
|
-
// behavior assumed a collision here, which minted a FALSE
|
|
854
|
-
// conflict for the most common fresh-install case: re-pushing a
|
|
855
|
-
// byte-identical \`core/\` scaffold file whose remote copy happened
|
|
856
|
-
// to be multipart-uploaded. Every fresh install that hit an
|
|
857
|
-
// already-populated bucket therefore came up "with conflicts".
|
|
858
|
-
//
|
|
859
|
-
// Instead, fetch the remote bytes once and compare content
|
|
860
|
-
// hashes directly — the same convergence guard the pull side
|
|
861
|
-
// uses (sync.ts). Identical content is NOT a conflict. On any
|
|
862
|
-
// fetch/hash failure we fail safe to "conflict" (false positives
|
|
863
|
-
// prompt the operator; false negatives risk clobbering a peer).
|
|
864
|
-
const remoteDiffers = await remoteContentDiffers(ctx, relativePath, localHash, hqRoot);
|
|
626
|
+
const remoteDiffers = await remoteContentDiffers(run.ctx, relativePath, localHash, run.hqRoot);
|
|
865
627
|
if (remoteDiffers) {
|
|
866
628
|
isFreshCollision = true;
|
|
867
629
|
}
|
|
868
630
|
else {
|
|
869
|
-
// Byte-identical multipart object already present. Seed the
|
|
870
|
-
// journal baseline from the remote so the next sync sees no
|
|
871
|
-
// change on either side, and skip the redundant PUT —
|
|
872
|
-
// re-uploading would needlessly rewrite remote and churn its
|
|
873
|
-
// etag from multipart to single-part.
|
|
874
631
|
multipartConverged = true;
|
|
875
632
|
}
|
|
876
633
|
}
|
|
877
634
|
}
|
|
878
635
|
if (multipartConverged) {
|
|
879
636
|
const lstat = fs.lstatSync(absolutePath);
|
|
880
|
-
updateEntry(journal, relativePath, localHash, lstat.size, "up", remoteMeta.etag, lstat.mtimeMs);
|
|
881
|
-
emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
882
|
-
filesSkipped++;
|
|
637
|
+
updateEntry(run.journal, relativePath, localHash, lstat.size, "up", remoteMeta.etag, lstat.mtimeMs);
|
|
638
|
+
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
639
|
+
counters.filesSkipped++;
|
|
883
640
|
return;
|
|
884
641
|
}
|
|
885
642
|
if ((localChanged && remoteChanged) || isFreshCollision) {
|
|
@@ -890,121 +647,53 @@ export async function share(options) {
|
|
|
890
647
|
remoteModified: remoteMeta.lastModified,
|
|
891
648
|
direction: "push",
|
|
892
649
|
});
|
|
893
|
-
emit({
|
|
650
|
+
run.emit({
|
|
894
651
|
type: "conflict",
|
|
895
652
|
path: relativePath,
|
|
896
653
|
direction: "push",
|
|
897
654
|
resolution,
|
|
898
655
|
});
|
|
899
656
|
if (resolution === "abort") {
|
|
900
|
-
// Flip the shared aborted flag — the pool drainer below sees this
|
|
901
|
-
// and stops queueing new items. In-flight tasks complete normally
|
|
902
|
-
// (S3 PUTs have no client-side cancel here). The outer abort
|
|
903
|
-
// return is built after the pool drains.
|
|
904
657
|
aborted = true;
|
|
905
658
|
abortFlightConflictPaths = [...conflictPaths];
|
|
906
659
|
return;
|
|
907
660
|
}
|
|
908
661
|
if (resolution === "keep" || resolution === "skip") {
|
|
909
|
-
// Bug #7 mirror branch: when the resolution is keep/skip on a
|
|
910
|
-
// FRESH collision (no prior journal entry), download the
|
|
911
|
-
// remote bytes to \`<orig>.conflict-<ts>-<short>\` so both
|
|
912
|
-
// versions survive on disk. Mirrors the pull-side mirror-write
|
|
913
|
-
// routine in sync.ts exactly. Skipped for stale-journal
|
|
914
|
-
// conflicts (the pre-Bug-#7 codepath) — those already produce
|
|
915
|
-
// a pull-side mirror on the next sync cycle.
|
|
916
662
|
if (isFreshCollision) {
|
|
917
|
-
|
|
918
|
-
const detectedAt = new Date().toISOString();
|
|
919
|
-
const machineId = readShortMachineId(hqRoot);
|
|
920
|
-
const originalRelative = path.relative(hqRoot, absolutePath);
|
|
921
|
-
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
922
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
923
|
-
await downloadFile(ctx, relativePath, conflictAbs);
|
|
924
|
-
appendConflictEntry(hqRoot, {
|
|
925
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
926
|
-
originalPath: originalRelative,
|
|
927
|
-
conflictPath: conflictRelative,
|
|
928
|
-
detectedAt,
|
|
929
|
-
side: "push",
|
|
930
|
-
machineId,
|
|
931
|
-
localHash,
|
|
932
|
-
remoteHash: normalizeEtag(remoteMeta.etag),
|
|
933
|
-
});
|
|
934
|
-
}
|
|
935
|
-
catch (mirrorErr) {
|
|
936
|
-
emit({
|
|
937
|
-
type: "error",
|
|
938
|
-
path: relativePath,
|
|
939
|
-
message: `conflict mirror write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
|
|
940
|
-
});
|
|
941
|
-
}
|
|
663
|
+
await writePushConflictMirror(run, item, normalizeEtag(remoteMeta.etag));
|
|
942
664
|
}
|
|
943
|
-
filesSkipped++;
|
|
665
|
+
counters.filesSkipped++;
|
|
944
666
|
return;
|
|
945
667
|
}
|
|
946
|
-
// "overwrite" falls through to upload
|
|
947
668
|
}
|
|
948
669
|
}
|
|
949
|
-
// Re-check abort flag right before the PUT — if a peer task aborted
|
|
950
|
-
// while we were waiting on HEAD, skip the PUT entirely. This is
|
|
951
|
-
// belt-and-suspenders alongside the queue-drain check; without it,
|
|
952
|
-
// up to TRANSFER_CONCURRENCY in-flight uploads could still issue PUTs
|
|
953
|
-
// after the user signaled abort.
|
|
954
670
|
if (aborted)
|
|
955
671
|
return;
|
|
956
|
-
// Conditional-write fence (storage-level backstop for the entire
|
|
957
|
-
// stale-clobber class). Every PUT asserts the remote state this pass
|
|
958
|
-
// just inspected:
|
|
959
|
-
// - remote exists → If-Match on the observed etag ("replace exactly
|
|
960
|
-
// what I HEAD'd"). Closes the HEAD→PUT TOCTOU: a peer's write
|
|
961
|
-
// landing in the window makes S3 itself reject with 412 instead of
|
|
962
|
-
// this machine silently regressing the object.
|
|
963
|
-
// - remote absent → If-None-Match:* ("create only"). If the HEAD was
|
|
964
|
-
// wrong about absence (any transport/state bug — the 2026-06-10..12
|
|
965
|
-
// regression storm's shape), the PUT 412s instead of clobbering.
|
|
966
|
-
// Enforced natively on the SDK transport; on the presigned transport it
|
|
967
|
-
// activates when files-presign signs the headers (see object-io.ts).
|
|
968
672
|
const precondition = remoteMeta
|
|
969
673
|
? { ifMatch: remoteMeta.etag }
|
|
970
674
|
: { ifNoneMatch: "*" };
|
|
971
|
-
// Upload — symlinks go through uploadSymlink (zero-byte body + target
|
|
972
|
-
// metadata), regular files through uploadFile (file contents). The
|
|
973
|
-
// discriminator is item.kind set by computePushPlan; both branches
|
|
974
|
-
// converge on the same journal/event update path below. Factored into a
|
|
975
|
-
// closure so the 412 "overwrite" resolution below can re-run it without
|
|
976
|
-
// the fence after explicit user consent.
|
|
977
675
|
const performUpload = async (pc) => {
|
|
978
676
|
const isSymlinkUpload = item.kind === "symlink";
|
|
979
|
-
// Capture lstat post-upload so size + mtimeMs stamped into the
|
|
980
|
-
// journal reflect the bytes we actually shipped. lstat (not stat)
|
|
981
|
-
// so a symlink stamps the link's own mtime, not the target's —
|
|
982
|
-
// matches the fast-path's lstat comparison so the next sync can
|
|
983
|
-
// skip without dereferencing.
|
|
984
677
|
const lstat = fs.lstatSync(absolutePath);
|
|
985
678
|
const size = isSymlinkUpload ? 0 : lstat.size;
|
|
986
679
|
const mtimeMs = lstat.mtimeMs;
|
|
987
680
|
const { etag } = isSymlinkUpload
|
|
988
|
-
? await uploadSymlink(ctx, item.target, relativePath, options.author, pc)
|
|
989
|
-
: await uploadFile(ctx, absolutePath, relativePath, options.author, pc);
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
if (message) {
|
|
996
|
-
journal.files[relativePath] = {
|
|
997
|
-
...journal.files[relativePath],
|
|
998
|
-
message,
|
|
681
|
+
? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
|
|
682
|
+
: await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
|
|
683
|
+
updateEntry(run.journal, relativePath, localHash, size, "up", etag, mtimeMs);
|
|
684
|
+
if (run.message) {
|
|
685
|
+
run.journal.files[relativePath] = {
|
|
686
|
+
...run.journal.files[relativePath],
|
|
687
|
+
message: run.message,
|
|
999
688
|
};
|
|
1000
689
|
}
|
|
1001
|
-
filesUploaded++;
|
|
1002
|
-
bytesUploaded += size;
|
|
1003
|
-
emit({
|
|
690
|
+
counters.filesUploaded++;
|
|
691
|
+
counters.bytesUploaded += size;
|
|
692
|
+
run.emit({
|
|
1004
693
|
type: "progress",
|
|
1005
694
|
path: relativePath,
|
|
1006
695
|
bytes: size,
|
|
1007
|
-
...(message ? { message } : {}),
|
|
696
|
+
...(run.message ? { message: run.message } : {}),
|
|
1008
697
|
});
|
|
1009
698
|
};
|
|
1010
699
|
try {
|
|
@@ -1012,17 +701,13 @@ export async function share(options) {
|
|
|
1012
701
|
}
|
|
1013
702
|
catch (err) {
|
|
1014
703
|
if (isPreconditionFailed(err)) {
|
|
1015
|
-
// The fence fired: the remote moved past (If-Match) or appeared at
|
|
1016
|
-
// (If-None-Match) this key between our HEAD and the PUT — exactly
|
|
1017
|
-
// the race the fence exists to catch. Surface as a push conflict;
|
|
1018
|
-
// never silently overwrite.
|
|
1019
704
|
conflictPaths.push(relativePath);
|
|
1020
705
|
const resolution = await resolveConflictSerialized({
|
|
1021
706
|
path: relativePath,
|
|
1022
707
|
localHash,
|
|
1023
708
|
direction: "push",
|
|
1024
709
|
});
|
|
1025
|
-
emit({
|
|
710
|
+
run.emit({
|
|
1026
711
|
type: "conflict",
|
|
1027
712
|
path: relativePath,
|
|
1028
713
|
direction: "push",
|
|
@@ -1034,13 +719,11 @@ export async function share(options) {
|
|
|
1034
719
|
return;
|
|
1035
720
|
}
|
|
1036
721
|
if (resolution === "overwrite") {
|
|
1037
|
-
// Explicit clobber consent — retry once without the fence. A
|
|
1038
|
-
// second failure falls through to the generic error emit.
|
|
1039
722
|
try {
|
|
1040
723
|
await performUpload(undefined);
|
|
1041
724
|
}
|
|
1042
725
|
catch (retryErr) {
|
|
1043
|
-
emit({
|
|
726
|
+
run.emit({
|
|
1044
727
|
type: "error",
|
|
1045
728
|
path: relativePath,
|
|
1046
729
|
message: retryErr instanceof Error ? retryErr.message : String(retryErr),
|
|
@@ -1048,41 +731,11 @@ export async function share(options) {
|
|
|
1048
731
|
}
|
|
1049
732
|
return;
|
|
1050
733
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
// branch above.
|
|
1054
|
-
try {
|
|
1055
|
-
const detectedAt = new Date().toISOString();
|
|
1056
|
-
const machineId = readShortMachineId(hqRoot);
|
|
1057
|
-
const originalRelative = path.relative(hqRoot, absolutePath);
|
|
1058
|
-
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
1059
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
1060
|
-
await downloadFile(ctx, relativePath, conflictAbs);
|
|
1061
|
-
appendConflictEntry(hqRoot, {
|
|
1062
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1063
|
-
originalPath: originalRelative,
|
|
1064
|
-
conflictPath: conflictRelative,
|
|
1065
|
-
detectedAt,
|
|
1066
|
-
side: "push",
|
|
1067
|
-
machineId,
|
|
1068
|
-
localHash,
|
|
1069
|
-
// remoteMeta (if any) predates the racing write that fired the
|
|
1070
|
-
// fence — record what we knew ("" when the key was believed
|
|
1071
|
-
// absent); the mirror file carries the authoritative remote bytes.
|
|
1072
|
-
remoteHash: remoteMeta ? normalizeEtag(remoteMeta.etag) : "",
|
|
1073
|
-
});
|
|
1074
|
-
}
|
|
1075
|
-
catch (mirrorErr) {
|
|
1076
|
-
emit({
|
|
1077
|
-
type: "error",
|
|
1078
|
-
path: relativePath,
|
|
1079
|
-
message: `conflict mirror write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
|
|
1080
|
-
});
|
|
1081
|
-
}
|
|
1082
|
-
filesSkipped++;
|
|
734
|
+
await writePushConflictMirror(run, item, remoteMeta ? normalizeEtag(remoteMeta.etag) : "");
|
|
735
|
+
counters.filesSkipped++;
|
|
1083
736
|
return;
|
|
1084
737
|
}
|
|
1085
|
-
emit({
|
|
738
|
+
run.emit({
|
|
1086
739
|
type: "error",
|
|
1087
740
|
path: relativePath,
|
|
1088
741
|
message: isAccessDenied(err)
|
|
@@ -1095,25 +748,6 @@ export async function share(options) {
|
|
|
1095
748
|
});
|
|
1096
749
|
}
|
|
1097
750
|
};
|
|
1098
|
-
// Drain the upload queue with bounded concurrency. Per-file progress
|
|
1099
|
-
// events fire from inside processUploadItem at file-settle time, so
|
|
1100
|
-
// cross-file event ordering is settle-order, not plan-walk-order — the
|
|
1101
|
-
// menubar's stream parser already tolerates per-company interleave so
|
|
1102
|
-
// this is shape-compatible. allSettled-style waiting (via Promise.race
|
|
1103
|
-
// on the in-flight set) keeps the pool topped up without idling on slow
|
|
1104
|
-
// members.
|
|
1105
|
-
//
|
|
1106
|
-
// Codex P1 (5.36.x): each worker promise is wrapped in a .catch that
|
|
1107
|
-
// records the error to `workerErrors` and resolves normally — so
|
|
1108
|
-
// `Promise.race(inFlight)` can never reject and unwind the drain loop
|
|
1109
|
-
// mid-flight. Without the wrap, an unhandled rejection inside
|
|
1110
|
-
// processUploadItem (e.g. headRemoteFile or refreshEntityContext, both
|
|
1111
|
-
// of which sit outside the per-item PUT try/catch) bubbled out of the
|
|
1112
|
-
// race, abandoned still-in-flight uploads that kept mutating remote
|
|
1113
|
-
// state, and skipped writeJournal — leaving remote and journal
|
|
1114
|
-
// permanently out of sync for the next run. After the pool fully
|
|
1115
|
-
// drains we throw an aggregated error so the caller still surfaces the
|
|
1116
|
-
// failure (and the journal reflects the writes that actually landed).
|
|
1117
751
|
const workerErrors = [];
|
|
1118
752
|
{
|
|
1119
753
|
const queue = [...uploadItems];
|
|
@@ -1134,87 +768,63 @@ export async function share(options) {
|
|
|
1134
768
|
await Promise.race(Array.from(inFlight));
|
|
1135
769
|
}
|
|
1136
770
|
else {
|
|
1137
|
-
// Aborted with nothing in flight → exit the drain loop.
|
|
1138
771
|
break;
|
|
1139
772
|
}
|
|
1140
773
|
}
|
|
1141
774
|
}
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
// shared array after the abort signal, and those should not show
|
|
1172
|
-
// up in the abort result.
|
|
1173
|
-
conflictPaths: abortFlightConflictPaths,
|
|
1174
|
-
aborted: true,
|
|
1175
|
-
};
|
|
775
|
+
return { aborted, abortFlightConflictPaths, workerErrors };
|
|
776
|
+
}
|
|
777
|
+
async function writePushConflictMirror(run, item, remoteHash) {
|
|
778
|
+
try {
|
|
779
|
+
const detectedAt = new Date().toISOString();
|
|
780
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
781
|
+
const originalRelative = path.relative(run.hqRoot, item.absolutePath);
|
|
782
|
+
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
783
|
+
const conflictAbs = path.join(run.hqRoot, conflictRelative);
|
|
784
|
+
if (!isMaterializationPathStillContained(run.syncRoot, conflictAbs)) {
|
|
785
|
+
run.emit({
|
|
786
|
+
type: "error",
|
|
787
|
+
path: item.relativePath,
|
|
788
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
else {
|
|
792
|
+
await downloadFile(run.ctx, item.relativePath, conflictAbs);
|
|
793
|
+
appendConflictEntry(run.hqRoot, {
|
|
794
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
795
|
+
originalPath: originalRelative,
|
|
796
|
+
conflictPath: conflictRelative,
|
|
797
|
+
detectedAt,
|
|
798
|
+
side: "push",
|
|
799
|
+
machineId,
|
|
800
|
+
localHash: item.localHash,
|
|
801
|
+
remoteHash,
|
|
802
|
+
});
|
|
803
|
+
}
|
|
1176
804
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
// intent only, and the dedupe at plan-computation time ensures we
|
|
1188
|
-
// don't double-issue for a key both plans matched.
|
|
1189
|
-
//
|
|
1190
|
-
// 2. `toTombstone` — the remote was 404 at HEAD time (cleaned up out
|
|
1191
|
-
// of band, e.g. someone hand-deleted via console). No DeleteObject
|
|
1192
|
-
// needed; just drop the journal entry so the journal converges with
|
|
1193
|
-
// reality. Emit a synthetic `progress` event with `deleted: true`
|
|
1194
|
-
// and bytes=0 so consumers see the convergence.
|
|
1195
|
-
//
|
|
1196
|
-
// 3. `refusedStale` — under `currency-gated`, the remote's current
|
|
1197
|
-
// ETag no longer matches the journal's recorded one. Some other
|
|
1198
|
-
// device modified the file since this device last synced it. Keep
|
|
1199
|
-
// the remote intact; keep the journal entry intact. The next pull
|
|
1200
|
-
// leg of `sync now` re-pulls naturally via the existing
|
|
1201
|
-
// `hasRemoteChanged` path. Emit a dedicated event so UIs can
|
|
1202
|
-
// surface the refusal without inferring it from absence.
|
|
1203
|
-
// Batch pre-mint DELETE URLs so a large delete set is ~ceil(N/1000) presign
|
|
1204
|
-
// calls, not N. No-op on the S3 SDK transport; best-effort.
|
|
805
|
+
catch (mirrorErr) {
|
|
806
|
+
run.emit({
|
|
807
|
+
type: "error",
|
|
808
|
+
path: item.relativePath,
|
|
809
|
+
message: "conflict mirror write failed: " +
|
|
810
|
+
(mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)),
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
async function executeDeletes(run, deletePlan, decommissionPlan, counters, filesRefusedStalePaths) {
|
|
1205
815
|
const deleteKeys = [...deletePlan.toDelete, ...decommissionPlan];
|
|
1206
|
-
await primeObjectTransport(ctx, "delete", deleteKeys);
|
|
816
|
+
await primeObjectTransport(run.ctx, "delete", deleteKeys);
|
|
1207
817
|
for (const relativePath of deleteKeys) {
|
|
1208
|
-
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
1209
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
818
|
+
if (run.vaultConfig && isExpiringSoon(run.ctx.expiresAt)) {
|
|
819
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
1210
820
|
}
|
|
1211
821
|
try {
|
|
1212
|
-
const entry = journal.files[relativePath];
|
|
822
|
+
const entry = run.journal.files[relativePath];
|
|
1213
823
|
const size = entry?.size ?? 0;
|
|
1214
|
-
await deleteRemoteFile(ctx, relativePath);
|
|
1215
|
-
removeEntry(journal, relativePath);
|
|
1216
|
-
filesDeleted++;
|
|
1217
|
-
emit({
|
|
824
|
+
await deleteRemoteFile(run.ctx, relativePath);
|
|
825
|
+
removeEntry(run.journal, relativePath);
|
|
826
|
+
counters.filesDeleted++;
|
|
827
|
+
run.emit({
|
|
1218
828
|
type: "progress",
|
|
1219
829
|
path: relativePath,
|
|
1220
830
|
bytes: size,
|
|
@@ -1222,7 +832,7 @@ export async function share(options) {
|
|
|
1222
832
|
});
|
|
1223
833
|
}
|
|
1224
834
|
catch (err) {
|
|
1225
|
-
emit({
|
|
835
|
+
run.emit({
|
|
1226
836
|
type: "error",
|
|
1227
837
|
path: relativePath,
|
|
1228
838
|
message: err instanceof Error ? err.message : String(err),
|
|
@@ -1230,9 +840,9 @@ export async function share(options) {
|
|
|
1230
840
|
}
|
|
1231
841
|
}
|
|
1232
842
|
for (const relativePath of deletePlan.toTombstone) {
|
|
1233
|
-
removeEntry(journal, relativePath);
|
|
1234
|
-
filesTombstoned++;
|
|
1235
|
-
emit({
|
|
843
|
+
removeEntry(run.journal, relativePath);
|
|
844
|
+
counters.filesTombstoned++;
|
|
845
|
+
run.emit({
|
|
1236
846
|
type: "progress",
|
|
1237
847
|
path: relativePath,
|
|
1238
848
|
bytes: 0,
|
|
@@ -1240,20 +850,15 @@ export async function share(options) {
|
|
|
1240
850
|
message: "tombstone (remote already 404)",
|
|
1241
851
|
});
|
|
1242
852
|
}
|
|
1243
|
-
// Decommission overrides refusedStale: a key whose peer drifted but
|
|
1244
|
-
// which the caller has claimed via `decommissionPrefixes` is processed
|
|
1245
|
-
// by the DeleteObject loop above; skip the refusal event here so the
|
|
1246
|
-
// event stream doesn't simultaneously claim "we deleted it" and "we
|
|
1247
|
-
// kept it on remote" for the same key.
|
|
1248
853
|
const decommissionedSet = decommissionPlan.length > 0 ? new Set(decommissionPlan) : null;
|
|
1249
854
|
for (const refused of deletePlan.refusedStale) {
|
|
1250
855
|
if (decommissionedSet && decommissionedSet.has(refused.key))
|
|
1251
856
|
continue;
|
|
1252
|
-
filesRefusedStale++;
|
|
857
|
+
counters.filesRefusedStale++;
|
|
1253
858
|
if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
|
|
1254
859
|
filesRefusedStalePaths.push(refused.key);
|
|
1255
860
|
}
|
|
1256
|
-
emit({
|
|
861
|
+
run.emit({
|
|
1257
862
|
type: "delete-refused-stale-etag",
|
|
1258
863
|
path: refused.key,
|
|
1259
864
|
journalEtag: refused.journalEtag,
|
|
@@ -1261,75 +866,65 @@ export async function share(options) {
|
|
|
1261
866
|
reason: refused.reason,
|
|
1262
867
|
});
|
|
1263
868
|
}
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
journal.lastSync = new Date().toISOString();
|
|
1267
|
-
writeJournal(journalSlug, journal);
|
|
1268
|
-
|
|
1269
|
-
// least one path was excluded. Sample is capped at 10 to keep the event
|
|
1270
|
-
// small (Set iteration order = insertion order, so samples are the first
|
|
1271
|
-
// ten paths encountered during the walk — deterministic, not random).
|
|
1272
|
-
if (excludedSet.size > 0) {
|
|
869
|
+
}
|
|
870
|
+
function finalizeShareJournal(run) {
|
|
871
|
+
run.journal.lastSync = new Date().toISOString();
|
|
872
|
+
writeJournal(run.journalSlug, run.journal);
|
|
873
|
+
if (run.excludedSet.size > 0) {
|
|
1273
874
|
const samplePaths = [];
|
|
1274
|
-
for (const p of excludedSet) {
|
|
875
|
+
for (const p of run.excludedSet) {
|
|
1275
876
|
samplePaths.push(p);
|
|
1276
877
|
if (samplePaths.length >= 10)
|
|
1277
878
|
break;
|
|
1278
879
|
}
|
|
1279
|
-
emit({
|
|
880
|
+
run.emit({
|
|
1280
881
|
type: "personal-vault-out-of-policy",
|
|
1281
|
-
count: excludedSet.size,
|
|
882
|
+
count: run.excludedSet.size,
|
|
1282
883
|
samplePaths,
|
|
1283
|
-
byId: { ...excludedById },
|
|
884
|
+
byId: { ...run.excludedById },
|
|
1284
885
|
});
|
|
1285
886
|
}
|
|
1286
|
-
|
|
1287
|
-
// paths fell outside the run's granted `prefixSet` and were skipped from the
|
|
1288
|
-
// upload/delete plan. Informational (NOT an error): the company still syncs
|
|
1289
|
-
// its in-scope subset and the run exits 0. Sample capped at 10 (insertion
|
|
1290
|
-
// order = walk order, deterministic).
|
|
1291
|
-
if (scopeExcludedSet.size > 0) {
|
|
887
|
+
if (run.scopeExcludedSet.size > 0) {
|
|
1292
888
|
const samplePaths = [];
|
|
1293
|
-
for (const p of scopeExcludedSet) {
|
|
889
|
+
for (const p of run.scopeExcludedSet) {
|
|
1294
890
|
samplePaths.push(p);
|
|
1295
891
|
if (samplePaths.length >= 10)
|
|
1296
892
|
break;
|
|
1297
893
|
}
|
|
1298
|
-
emit({
|
|
894
|
+
run.emit({
|
|
1299
895
|
type: "scope-excluded",
|
|
1300
|
-
count: scopeExcludedSet.size,
|
|
896
|
+
count: run.scopeExcludedSet.size,
|
|
1301
897
|
samplePaths,
|
|
1302
898
|
});
|
|
1303
899
|
}
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
// outside the per-item PUT try/catch), we deliberately let the pool
|
|
1307
|
-
// drain AND let post-pool stages (deletes, tombstones, journal write,
|
|
1308
|
-
// personal-vault summary) run to completion so journal + remote stay
|
|
1309
|
-
// converged on what actually landed. NOW surface the failure to the
|
|
1310
|
-
// caller — preserving the first error's stack so debugging works.
|
|
1311
|
-
// Aggregate count is reported in the message for visibility when
|
|
1312
|
-
// multiple workers failed.
|
|
900
|
+
}
|
|
901
|
+
function throwUploadWorkerErrors(workerErrors) {
|
|
1313
902
|
if (workerErrors.length > 0) {
|
|
1314
903
|
const first = workerErrors[0];
|
|
1315
904
|
if (workerErrors.length > 1) {
|
|
1316
|
-
first.message =
|
|
905
|
+
first.message =
|
|
906
|
+
first.message +
|
|
907
|
+
" (and " +
|
|
908
|
+
(workerErrors.length - 1) +
|
|
909
|
+
" more upload-worker errors)";
|
|
1317
910
|
}
|
|
1318
911
|
throw first;
|
|
1319
912
|
}
|
|
913
|
+
}
|
|
914
|
+
function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths, aborted) {
|
|
1320
915
|
return {
|
|
1321
|
-
filesUploaded,
|
|
1322
|
-
bytesUploaded,
|
|
1323
|
-
filesSkipped,
|
|
1324
|
-
filesDeleted,
|
|
1325
|
-
filesTombstoned,
|
|
1326
|
-
filesRefusedStale,
|
|
916
|
+
filesUploaded: counters.filesUploaded,
|
|
917
|
+
bytesUploaded: counters.bytesUploaded,
|
|
918
|
+
filesSkipped: counters.filesSkipped,
|
|
919
|
+
filesDeleted: counters.filesDeleted,
|
|
920
|
+
filesTombstoned: counters.filesTombstoned,
|
|
921
|
+
filesRefusedStale: counters.filesRefusedStale,
|
|
1327
922
|
filesRefusedStalePaths,
|
|
1328
|
-
filesSuppressedByTombstone,
|
|
1329
|
-
filesExcludedByPolicy: excludedSet.size,
|
|
1330
|
-
filesExcludedByScope: scopeExcludedSet.size,
|
|
923
|
+
filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
|
|
924
|
+
filesExcludedByPolicy: run.excludedSet.size,
|
|
925
|
+
filesExcludedByScope: run.scopeExcludedSet.size,
|
|
1331
926
|
conflictPaths,
|
|
1332
|
-
aborted
|
|
927
|
+
aborted,
|
|
1333
928
|
};
|
|
1334
929
|
}
|
|
1335
930
|
/**
|
|
@@ -1389,22 +984,6 @@ function defaultConsoleLogger(event) {
|
|
|
1389
984
|
` To proceed anyway: re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1 (or propagateDeletePolicy:"all").`);
|
|
1390
985
|
}
|
|
1391
986
|
}
|
|
1392
|
-
/**
|
|
1393
|
-
* Resolve active company from .hq/config.json or parent directory chain.
|
|
1394
|
-
*/
|
|
1395
|
-
function resolveActiveCompany(hqRoot) {
|
|
1396
|
-
const configPath = path.join(hqRoot, ".hq", "config.json");
|
|
1397
|
-
if (fs.existsSync(configPath)) {
|
|
1398
|
-
try {
|
|
1399
|
-
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
1400
|
-
return config.activeCompany ?? config.companySlug;
|
|
1401
|
-
}
|
|
1402
|
-
catch {
|
|
1403
|
-
// Ignore parse errors
|
|
1404
|
-
}
|
|
1405
|
-
}
|
|
1406
|
-
return undefined;
|
|
1407
|
-
}
|
|
1408
987
|
/**
|
|
1409
988
|
* Collect files from paths (expanding directories recursively).
|
|
1410
989
|
*
|
|
@@ -1564,6 +1143,10 @@ function isWithin(parent, child) {
|
|
|
1564
1143
|
const rel = path.relative(parentCanon, childCanon);
|
|
1565
1144
|
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
1566
1145
|
}
|
|
1146
|
+
function isPathWithin(parent, child) {
|
|
1147
|
+
const rel = path.relative(parent, child);
|
|
1148
|
+
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
1149
|
+
}
|
|
1567
1150
|
function realpathSafe(p) {
|
|
1568
1151
|
try {
|
|
1569
1152
|
return fs.realpathSync.native(p);
|
|
@@ -1572,6 +1155,49 @@ function realpathSafe(p) {
|
|
|
1572
1155
|
return p;
|
|
1573
1156
|
}
|
|
1574
1157
|
}
|
|
1158
|
+
function deepestExistingAncestor(start) {
|
|
1159
|
+
let current = start;
|
|
1160
|
+
for (;;) {
|
|
1161
|
+
try {
|
|
1162
|
+
fs.lstatSync(current);
|
|
1163
|
+
return current;
|
|
1164
|
+
}
|
|
1165
|
+
catch (err) {
|
|
1166
|
+
const code = err && typeof err === "object" && "code" in err
|
|
1167
|
+
? err.code
|
|
1168
|
+
: undefined;
|
|
1169
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
1170
|
+
return null;
|
|
1171
|
+
}
|
|
1172
|
+
const parent = path.dirname(current);
|
|
1173
|
+
if (parent === current)
|
|
1174
|
+
return null;
|
|
1175
|
+
current = parent;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
function isMaterializationPathStillContained(root, localPath) {
|
|
1179
|
+
const resolvedRoot = path.resolve(root);
|
|
1180
|
+
const resolvedLocal = path.resolve(localPath);
|
|
1181
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal))
|
|
1182
|
+
return false;
|
|
1183
|
+
let realRoot;
|
|
1184
|
+
try {
|
|
1185
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1186
|
+
}
|
|
1187
|
+
catch {
|
|
1188
|
+
return false;
|
|
1189
|
+
}
|
|
1190
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1191
|
+
if (existingAncestor === null)
|
|
1192
|
+
return false;
|
|
1193
|
+
try {
|
|
1194
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1195
|
+
return isPathWithin(realRoot, realAncestor);
|
|
1196
|
+
}
|
|
1197
|
+
catch {
|
|
1198
|
+
return false;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1575
1201
|
/**
|
|
1576
1202
|
* Containment check tailored for symlinks. Canonicalizes the link's
|
|
1577
1203
|
* PARENT DIR (which is a real dir, not the link), then compares the
|
|
@@ -1593,20 +1219,6 @@ function isWithinForLink(parent, linkPath) {
|
|
|
1593
1219
|
const rel = path.relative(parentReal, candidate);
|
|
1594
1220
|
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
1595
1221
|
}
|
|
1596
|
-
/**
|
|
1597
|
-
* Returns true when the remote object appears to have moved since the
|
|
1598
|
-
* journal entry's last-recorded sync. Prefers ETag equality; falls back to
|
|
1599
|
-
* `lastModified > syncedAt` for legacy entries written before remoteEtag
|
|
1600
|
-
* was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
|
|
1601
|
-
* S3-side mtime that exactly equals our syncedAt is not treated as drift.
|
|
1602
|
-
*/
|
|
1603
|
-
function hasRemoteChanged(remote, entry) {
|
|
1604
|
-
if (entry.remoteEtag) {
|
|
1605
|
-
return normalizeEtag(remote.etag) !== entry.remoteEtag;
|
|
1606
|
-
}
|
|
1607
|
-
const syncedAt = new Date(entry.syncedAt).getTime();
|
|
1608
|
-
return remote.lastModified.getTime() > syncedAt;
|
|
1609
|
-
}
|
|
1610
1222
|
/**
|
|
1611
1223
|
* Resolve each user-supplied share path to a directory under `syncRoot`,
|
|
1612
1224
|
* returning the company-relative prefix that constrains delete propagation.
|
|
@@ -1915,6 +1527,10 @@ async function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSync, poli
|
|
|
1915
1527
|
// independently — one failed HEAD doesn't poison the others (errors
|
|
1916
1528
|
// propagate from the chunk's Promise.all and are surfaced by share()'s
|
|
1917
1529
|
// outer try/catch, mirroring the existing pre-share error handling).
|
|
1530
|
+
// Warm the GET presigns the HEAD pass reuses so a large candidate set doesn't
|
|
1531
|
+
// mint one presign per HEAD and trip the presign breaker. Mirrors the
|
|
1532
|
+
// new-files + tombstone HEAD-pass pre-primes.
|
|
1533
|
+
await primeObjectTransport(ctx, "get", headCandidates.map((c) => c.key));
|
|
1918
1534
|
for (let i = 0; i < headCandidates.length; i += DELETE_PLAN_HEAD_CONCURRENCY) {
|
|
1919
1535
|
const chunk = headCandidates.slice(i, i + DELETE_PLAN_HEAD_CONCURRENCY);
|
|
1920
1536
|
const results = await Promise.all(chunk.map(async (c) => ({
|