@agent-watch-ai/edge 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +6 -5
- package/dist/events/constants/events.constants.d.ts +5 -2
- package/dist/events/constants/events.constants.js +4 -1
- package/dist/events/index.d.ts +1 -0
- package/dist/events/product-event.d.ts +7 -4
- package/dist/events/types/repo-snapshot.types.d.ts +1 -4
- package/dist/events/types/repo-tree.types.d.ts +24 -0
- package/dist/events/types/repo-tree.types.js +1 -0
- package/dist/git/constants/tree.constants.d.ts +71 -0
- package/dist/git/constants/tree.constants.js +92 -0
- package/dist/git/git-context.d.ts +1 -0
- package/dist/git/git-context.js +3 -2
- package/dist/git/repo-tree.d.ts +69 -0
- package/dist/git/repo-tree.js +130 -0
- package/dist/git/types/git.types.d.ts +5 -2
- package/dist/git/types/snapshot.types.d.ts +12 -0
- package/dist/pipeline/hook-pipeline.js +40 -24
- package/dist/privacy/product-capture.js +34 -5
- package/dist/snapshot/branch-selection.js +3 -1
- package/dist/snapshot/budget.js +2 -2
- package/dist/snapshot/constants/snapshot-stages.constants.d.ts +8 -0
- package/dist/snapshot/constants/snapshot-stages.constants.js +8 -0
- package/dist/snapshot/constants/snapshot.constants.d.ts +6 -0
- package/dist/snapshot/constants/snapshot.constants.js +6 -0
- package/dist/snapshot/index.d.ts +5 -0
- package/dist/snapshot/index.js +2 -0
- package/dist/snapshot/snapshot-event.d.ts +7 -0
- package/dist/snapshot/snapshot-event.js +8 -5
- package/dist/snapshot/snapshot-state.js +19 -1
- package/dist/snapshot/tree-event.d.ts +14 -0
- package/dist/snapshot/tree-event.js +91 -0
- package/dist/snapshot/tree-pipeline.d.ts +12 -0
- package/dist/snapshot/tree-pipeline.js +153 -0
- package/dist/snapshot/types/snapshot.types.d.ts +44 -1
- package/dist/transport/constants/transport.constants.d.ts +5 -0
- package/dist/transport/constants/transport.constants.js +5 -0
- package/dist/transport/queue.js +40 -6
- package/dist/turns/turn-summary.js +3 -4
- package/dist/turns/types/turn-summary.types.d.ts +3 -5
- package/docs/DATA_HANDLING.md +77 -26
- package/docs/PERFORMANCE.md +17 -0
- package/package.json +1 -1
|
@@ -7,7 +7,7 @@ import { next, runFlow, step, stop } from '../core/pipe.js';
|
|
|
7
7
|
import { loadEffectiveConfig } from '../config/repo-config.js';
|
|
8
8
|
import { CREDENTIAL_BUDGET_SHARE, NOT_SIGNED_IN } from '../credentials/constants/credentials.constants.js';
|
|
9
9
|
import { cachedBearer, resolveBearer } from '../credentials/credentials.js';
|
|
10
|
-
import { awaitingSignIn, deviceFor, hasCredential, identityOf } from '../credentials/identity.js';
|
|
10
|
+
import { awaitingSignIn, deviceFor, hasCredential, identityFingerprint, identityOf } from '../credentials/identity.js';
|
|
11
11
|
import { cachedAccessToken } from '../credentials/token-refresher.js';
|
|
12
12
|
import { credentialScope, inheritedRootConfig, selectRoot, servesMultipleIdentities } from '../config/root-config.js';
|
|
13
13
|
import { DECISION_BLOCK } from '../enforcement/constants/enforcement.constants.js';
|
|
@@ -22,6 +22,7 @@ import { rewriteNativeBearers } from '../providers/shared/bearer-rewrite.js';
|
|
|
22
22
|
import { isUninstalled } from '../storage/uninstalled.js';
|
|
23
23
|
import { shellDirectories } from '../providers/shared/shell-directories.js';
|
|
24
24
|
import { runSnapshotPipeline } from '../snapshot/snapshot-pipeline.js';
|
|
25
|
+
import { runTreePipeline } from '../snapshot/tree-pipeline.js';
|
|
25
26
|
import { SnapshotStateStore } from '../snapshot/snapshot-state.js';
|
|
26
27
|
import { SNAPSHOT_BUDGET_MS } from '../snapshot/constants/snapshot.constants.js';
|
|
27
28
|
import { BackendAuthBlock } from '../transport/auth-block.js';
|
|
@@ -575,31 +576,46 @@ async function snapshot(state) {
|
|
|
575
576
|
const capture = state.workCapture ?? state.config.capture;
|
|
576
577
|
if (!repository || state.summary?.work_evidence === 'inherited' || !capture.git)
|
|
577
578
|
return stop(state, STOP_NO_SNAPSHOT);
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
579
|
+
const input = {
|
|
580
|
+
// The checkout the summary named, which need not be the folder the agent sits in.
|
|
581
|
+
cwd: state.workRoot ?? state.cwd,
|
|
582
|
+
// The repository's own policy, which may lower the global one: file names
|
|
583
|
+
// off means the files listing never runs, not that it is emptied later.
|
|
584
|
+
files: capture.files !== false,
|
|
585
|
+
// A digest, never the credential: the tree is owed once per repository and identity.
|
|
586
|
+
identity: treeIdentity(state),
|
|
587
|
+
repository,
|
|
588
|
+
provider: state.summary?.provider ?? state.provider.id,
|
|
589
|
+
surface: state.summary?.surface ?? state.provider.id,
|
|
590
|
+
agentName: state.summary?.agent.name ?? state.provider.id,
|
|
591
|
+
developerId: state.summary?.developer_id,
|
|
592
|
+
installationId: state.config.installationId,
|
|
593
|
+
sessionId: state.summary?.session_id,
|
|
594
|
+
capturedAt: state.env.now().toISOString(),
|
|
595
|
+
// Real time, deliberately, where the line above uses the injected clock:
|
|
596
|
+
// this bounds subprocess timeouts, which are measured against the wall.
|
|
597
|
+
deadline: Date.now() + SNAPSHOT_BUDGET_MS,
|
|
598
|
+
run: runGit
|
|
599
|
+
};
|
|
600
|
+
const store = new SnapshotStateStore(state.paths.snapshotsDir);
|
|
601
|
+
const queue = buildQueue(state);
|
|
602
|
+
await runSnapshotPipeline({ input, store, queue });
|
|
603
|
+
// The trunk's paths and declaration names, once per repository and identity,
|
|
604
|
+
// inside the same deadline. File names off means no listing at all.
|
|
605
|
+
if (input.files)
|
|
606
|
+
await runTreePipeline({ input, store, queue });
|
|
601
607
|
return next(state);
|
|
602
608
|
}
|
|
609
|
+
/**
|
|
610
|
+
* The identity a tree is recorded as sent for: a re-setup with a new credential owes it again.
|
|
611
|
+
*
|
|
612
|
+
* @param state - Current flow state.
|
|
613
|
+
* @returns The identity's on-disk fingerprint, or undefined with no credential.
|
|
614
|
+
*/
|
|
615
|
+
function treeIdentity(state) {
|
|
616
|
+
const identity = identityOf(state.config);
|
|
617
|
+
return identity ? identityFingerprint(identity) : undefined;
|
|
618
|
+
}
|
|
603
619
|
/**
|
|
604
620
|
* The offline queue for this run, partitioned by the identity it sends as.
|
|
605
621
|
* `deliver` settles a pre-partition backlog before the first build.
|
|
@@ -16,16 +16,21 @@ export function applyProductCapture(event, capture) {
|
|
|
16
16
|
if (event.event.type === 'repo.snapshot') {
|
|
17
17
|
if (capture?.git === false)
|
|
18
18
|
return undefined;
|
|
19
|
-
|
|
19
|
+
const snapshot = withoutUnreadSnapshotFields(event);
|
|
20
|
+
return sanitizeValue(capture?.files === false ? withoutCommitFiles(snapshot) : snapshot);
|
|
20
21
|
}
|
|
22
|
+
// A tree is nothing but paths and the names declared in them: it goes with
|
|
23
|
+
// either flag, queued records included, and is never sent emptied.
|
|
24
|
+
if (event.event.type === 'repo.tree')
|
|
25
|
+
return capture?.git === false || capture?.files === false ? undefined : sanitizeValue(event);
|
|
21
26
|
if (event.event.type !== 'turn.summary')
|
|
22
27
|
return sanitizeValue(event);
|
|
23
28
|
const summary = event;
|
|
24
29
|
// Response text goes unconditionally: this release never builds it, but a
|
|
25
30
|
// summary queued by an older one can still carry it. `prompt` is that older
|
|
26
31
|
// release's field and goes the same way; prompt text now rides only as
|
|
27
|
-
// `prompt_text`.
|
|
28
|
-
//
|
|
32
|
+
// `prompt_text`. The prompt and response evidence (length + sha256) goes
|
|
33
|
+
// too: nothing in the platform reads it.
|
|
29
34
|
//
|
|
30
35
|
// The content fields need an explicit `true`: they are opt-in, and a
|
|
31
36
|
// missing policy is not consent. The path and repository fields need an
|
|
@@ -36,6 +41,8 @@ export function applyProductCapture(event, capture) {
|
|
|
36
41
|
return sanitizeValue({
|
|
37
42
|
...event,
|
|
38
43
|
prompt: undefined,
|
|
44
|
+
prompt_evidence: undefined,
|
|
45
|
+
response_evidence: undefined,
|
|
39
46
|
tools_used: capture?.mcpNames === false ? foldMcpNames(summary.tools_used) : summary.tools_used,
|
|
40
47
|
response: undefined,
|
|
41
48
|
prompt_text: capture?.promptText === true ? summary.prompt_text : undefined,
|
|
@@ -46,14 +53,15 @@ export function applyProductCapture(event, capture) {
|
|
|
46
53
|
branch: git ? undefined : summary.branch,
|
|
47
54
|
commit: git ? undefined : summary.commit,
|
|
48
55
|
jira_ids: git ? undefined : summary.jira_ids,
|
|
49
|
-
prompt_jira_ids: git ? undefined : summary.prompt_jira_ids,
|
|
56
|
+
prompt_jira_ids: git ? undefined : summary.prompt_jira_ids?.slice(0, 1),
|
|
50
57
|
work_evidence: git ? undefined : summary.work_evidence,
|
|
51
58
|
parent_session_id: git ? undefined : summary.parent_session_id,
|
|
52
59
|
// Content-free (ids, times, outcome, digests), but on by default with git
|
|
53
60
|
// context and withheld whole without it: a digest of a command still says
|
|
54
61
|
// which command ran when.
|
|
55
62
|
tool_events: git ? undefined : withMcpPolicy(summary.tool_events, capture?.mcpNames === false),
|
|
56
|
-
|
|
63
|
+
// The blob id is what is matched; the path beside it in older records is not read.
|
|
64
|
+
content_hashes: files ? undefined : summary.content_hashes?.map(({ blob_sha }) => ({ blob_sha })),
|
|
57
65
|
manifest_repository: files ? undefined : summary.manifest_repository,
|
|
58
66
|
files_changed: files ? undefined : summary.files_changed,
|
|
59
67
|
files_touched: files ? undefined : summary.files_touched,
|
|
@@ -76,6 +84,27 @@ function withMcpPolicy(events, fold) {
|
|
|
76
84
|
return events;
|
|
77
85
|
return events.map((event) => ({ ...event, tool: foldMcpName(event.tool) }));
|
|
78
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* A snapshot without the fields nothing in the platform reads: branch
|
|
89
|
+
* `last_commit_at`, commit `authored_at` and each file's status letter.
|
|
90
|
+
* The builder no longer sets them; a snapshot an older release queued
|
|
91
|
+
* loses them here.
|
|
92
|
+
*
|
|
93
|
+
* @param event - Snapshot, possibly from an older installation.
|
|
94
|
+
* @returns A copy without them.
|
|
95
|
+
*/
|
|
96
|
+
function withoutUnreadSnapshotFields(event) {
|
|
97
|
+
return {
|
|
98
|
+
...event,
|
|
99
|
+
branches: event.branches.map(({ last_commit_at: _lastCommitAt, ...branch }) => ({
|
|
100
|
+
...branch,
|
|
101
|
+
commits: branch.commits.map(({ authored_at: _authoredAt, ...commit }) => ({
|
|
102
|
+
...commit,
|
|
103
|
+
files: commit.files?.map(({ path }) => ({ path }))
|
|
104
|
+
}))
|
|
105
|
+
}))
|
|
106
|
+
};
|
|
107
|
+
}
|
|
79
108
|
/**
|
|
80
109
|
* A snapshot with no commit file names, for a policy that withholds paths.
|
|
81
110
|
*
|
|
@@ -62,8 +62,10 @@ export function nextSnapshotState(input, sent) {
|
|
|
62
62
|
// deltas were never recomputed, and they would sit on the wrong base until
|
|
63
63
|
// their heads happened to move.
|
|
64
64
|
const described = input.refs.every((ref) => sentNames.has(ref.name));
|
|
65
|
+
// The tree is the tree flow's to write; a branch send carries it over untouched.
|
|
65
66
|
return {
|
|
66
67
|
defaultBranch: described ? input.defaultBranch : input.stored.defaultBranch,
|
|
67
|
-
branches
|
|
68
|
+
branches,
|
|
69
|
+
...(input.stored.tree ? { tree: input.stored.tree } : {})
|
|
68
70
|
};
|
|
69
71
|
}
|
package/dist/snapshot/budget.js
CHANGED
|
@@ -27,11 +27,11 @@ export function remainingMs(deadline, now = Date.now) {
|
|
|
27
27
|
* @returns A runner whose timeout is whatever is left, capped at the git default.
|
|
28
28
|
*/
|
|
29
29
|
export function budgetedRunner(run, deadline, now = Date.now) {
|
|
30
|
-
return async (args, cwd, timeoutMs, home) => {
|
|
30
|
+
return async (args, cwd, timeoutMs, home, maxBufferBytes) => {
|
|
31
31
|
const left = remainingMs(deadline, now);
|
|
32
32
|
if (left === 0)
|
|
33
33
|
return undefined;
|
|
34
|
-
return run(args, cwd, Math.min(timeoutMs || GIT_TIMEOUT_MS, left), home);
|
|
34
|
+
return run(args, cwd, Math.min(timeoutMs || GIT_TIMEOUT_MS, left), home, maxBufferBytes);
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
37
|
/**
|
|
@@ -9,3 +9,11 @@ export declare const STOP_NOT_A_REPOSITORY = "no branches: not a repository, or
|
|
|
9
9
|
export declare const STOP_NOTHING_CHANGED = "no branch moved and none is due a refresh";
|
|
10
10
|
export declare const STOP_BUDGET_SPENT = "snapshot budget spent";
|
|
11
11
|
export declare const STOP_NO_EVENT = "no snapshot event was built";
|
|
12
|
+
/** The tree flow's stages and stops. */
|
|
13
|
+
export declare const STAGE_TREE_DUE = "tree-due";
|
|
14
|
+
export declare const STAGE_TREE_RESOLVE = "tree-resolve";
|
|
15
|
+
export declare const STAGE_TREE_COLLECT = "tree-collect";
|
|
16
|
+
export declare const STOP_TREE_SENT = "tree already sent for this repository and identity";
|
|
17
|
+
export declare const STOP_TREE_NOT_DUE = "tree attempt failed within the last hour";
|
|
18
|
+
export declare const STOP_TREE_NO_TRUNK = "no default branch tree to list";
|
|
19
|
+
export declare const STOP_TREE_UNLISTED = "tree could not be listed";
|
|
@@ -9,3 +9,11 @@ export const STOP_NOT_A_REPOSITORY = 'no branches: not a repository, or it has n
|
|
|
9
9
|
export const STOP_NOTHING_CHANGED = 'no branch moved and none is due a refresh';
|
|
10
10
|
export const STOP_BUDGET_SPENT = 'snapshot budget spent';
|
|
11
11
|
export const STOP_NO_EVENT = 'no snapshot event was built';
|
|
12
|
+
/** The tree flow's stages and stops. */
|
|
13
|
+
export const STAGE_TREE_DUE = 'tree-due';
|
|
14
|
+
export const STAGE_TREE_RESOLVE = 'tree-resolve';
|
|
15
|
+
export const STAGE_TREE_COLLECT = 'tree-collect';
|
|
16
|
+
export const STOP_TREE_SENT = 'tree already sent for this repository and identity';
|
|
17
|
+
export const STOP_TREE_NOT_DUE = 'tree attempt failed within the last hour';
|
|
18
|
+
export const STOP_TREE_NO_TRUNK = 'no default branch tree to list';
|
|
19
|
+
export const STOP_TREE_UNLISTED = 'tree could not be listed';
|
|
@@ -86,3 +86,9 @@ export declare const SNAPSHOT_MAX_PATH_LENGTH = 1024;
|
|
|
86
86
|
export declare const SNAPSHOT_RE_CONTROL_CHARACTER: RegExp;
|
|
87
87
|
/** Hex characters kept from the repository digest that names a state file. */
|
|
88
88
|
export declare const SNAPSHOT_STATE_HASH_LENGTH = 32;
|
|
89
|
+
/**
|
|
90
|
+
* How long a repository whose `repo.tree` attempt failed (no trunk, listing
|
|
91
|
+
* failed, budget spent) waits before the next one. The tree is sent once per
|
|
92
|
+
* repository and identity; this only bounds the cost of not managing to.
|
|
93
|
+
*/
|
|
94
|
+
export declare const TREE_RETRY_INTERVAL_MS: number;
|
|
@@ -87,3 +87,9 @@ export const SNAPSHOT_MAX_PATH_LENGTH = 1024;
|
|
|
87
87
|
export const SNAPSHOT_RE_CONTROL_CHARACTER = /[\x00-\x1f\x7f]/;
|
|
88
88
|
/** Hex characters kept from the repository digest that names a state file. */
|
|
89
89
|
export const SNAPSHOT_STATE_HASH_LENGTH = 32;
|
|
90
|
+
/**
|
|
91
|
+
* How long a repository whose `repo.tree` attempt failed (no trunk, listing
|
|
92
|
+
* failed, budget spent) waits before the next one. The tree is sent once per
|
|
93
|
+
* repository and identity; this only bounds the cost of not managing to.
|
|
94
|
+
*/
|
|
95
|
+
export const TREE_RETRY_INTERVAL_MS = 60 * 60 * 1000;
|
package/dist/snapshot/index.d.ts
CHANGED
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
* layer is an overlay on work items, and a team with no tracker connected has
|
|
7
7
|
* none — the branches on this machine are then the only evidence that a piece
|
|
8
8
|
* of work exists at all.
|
|
9
|
+
*
|
|
10
|
+
* The tree flow beside it sends the trunk's source paths and declaration names
|
|
11
|
+
* (`repo.tree`) once per repository and identity, for a tenant whose code nothing may clone.
|
|
9
12
|
*/
|
|
10
13
|
export type { BuildSnapshotInput, SelectionInput, SnapshotFlowInput, SnapshotFlowState, SnapshotIdentity, SnapshotPipelineInput, SnapshotQueue, SnapshotState, SnapshotStateEntry, SnapshotStateReader } from './types/snapshot.types.js';
|
|
11
14
|
export { SNAPSHOT_BRANCH_COUNT, SNAPSHOT_BUDGET_MS, SNAPSHOT_COMMITS_PER_BRANCH, SNAPSHOT_REFRESH_MS } from './constants/snapshot.constants.js';
|
|
@@ -13,4 +16,6 @@ export { nextSnapshotState, selectChangedBranches } from './branch-selection.js'
|
|
|
13
16
|
export { budgetedRunner, remainingMs, withinBudget } from './budget.js';
|
|
14
17
|
export { buildRepoSnapshot } from './snapshot-event.js';
|
|
15
18
|
export { runSnapshotPipeline } from './snapshot-pipeline.js';
|
|
19
|
+
export { buildRepoTree } from './tree-event.js';
|
|
20
|
+
export { runTreePipeline } from './tree-pipeline.js';
|
|
16
21
|
export { SnapshotStateStore, emptyState } from './snapshot-state.js';
|
package/dist/snapshot/index.js
CHANGED
|
@@ -3,4 +3,6 @@ export { nextSnapshotState, selectChangedBranches } from './branch-selection.js'
|
|
|
3
3
|
export { budgetedRunner, remainingMs, withinBudget } from './budget.js';
|
|
4
4
|
export { buildRepoSnapshot } from './snapshot-event.js';
|
|
5
5
|
export { runSnapshotPipeline } from './snapshot-pipeline.js';
|
|
6
|
+
export { buildRepoTree } from './tree-event.js';
|
|
7
|
+
export { runTreePipeline } from './tree-pipeline.js';
|
|
6
8
|
export { SnapshotStateStore, emptyState } from './snapshot-state.js';
|
|
@@ -15,3 +15,10 @@ import type { BuildSnapshotInput } from './types/snapshot.types.js';
|
|
|
15
15
|
* @returns The snapshot event.
|
|
16
16
|
*/
|
|
17
17
|
export declare function buildRepoSnapshot(input: BuildSnapshotInput): RepoSnapshotEvent;
|
|
18
|
+
/**
|
|
19
|
+
* Whether a path may be named at all: not over-long, no control characters, not a credential file.
|
|
20
|
+
*
|
|
21
|
+
* @param filePath - Repository-relative path.
|
|
22
|
+
* @returns True when the path may be sent.
|
|
23
|
+
*/
|
|
24
|
+
export declare function sendable(filePath: string): boolean;
|
|
@@ -49,11 +49,9 @@ export function buildRepoSnapshot(input) {
|
|
|
49
49
|
return compact({
|
|
50
50
|
name: branch.name,
|
|
51
51
|
head_sha: branch.headSha,
|
|
52
|
-
last_commit_at: branch.lastCommitAt,
|
|
53
52
|
commits: branch.commits.map((commit, index) => compact({
|
|
54
53
|
sha: commit.sha,
|
|
55
54
|
subject: commit.subject.slice(0, SNAPSHOT_MAX_SUBJECT_LENGTH),
|
|
56
|
-
authored_at: commit.authoredAt,
|
|
57
55
|
files: files[index]
|
|
58
56
|
}))
|
|
59
57
|
});
|
|
@@ -84,15 +82,20 @@ function branchFiles(commits) {
|
|
|
84
82
|
continue;
|
|
85
83
|
files -= 1;
|
|
86
84
|
bytes -= size;
|
|
87
|
-
kept.push(file);
|
|
85
|
+
kept.push({ path: file.path });
|
|
88
86
|
}
|
|
89
87
|
// Empty only when git listed nothing: a commit whose every file was withheld
|
|
90
88
|
// or capped did change something, and an empty list would say it had not.
|
|
91
89
|
return kept.length > 0 || commit.files?.length === 0 ? kept : undefined;
|
|
92
90
|
});
|
|
93
91
|
}
|
|
94
|
-
/**
|
|
95
|
-
|
|
92
|
+
/**
|
|
93
|
+
* Whether a path may be named at all: not over-long, no control characters, not a credential file.
|
|
94
|
+
*
|
|
95
|
+
* @param filePath - Repository-relative path.
|
|
96
|
+
* @returns True when the path may be sent.
|
|
97
|
+
*/
|
|
98
|
+
export function sendable(filePath) {
|
|
96
99
|
return filePath.length <= SNAPSHOT_MAX_PATH_LENGTH && !SNAPSHOT_RE_CONTROL_CHARACTER.test(filePath) && !RE_SECRET_FILE.test(path.posix.basename(filePath));
|
|
97
100
|
}
|
|
98
101
|
/**
|
|
@@ -42,7 +42,8 @@ export class SnapshotStateStore {
|
|
|
42
42
|
return emptyState();
|
|
43
43
|
return {
|
|
44
44
|
defaultBranch: typeof record.defaultBranch === 'string' ? record.defaultBranch : undefined,
|
|
45
|
-
branches: readEntries(record.branches)
|
|
45
|
+
branches: readEntries(record.branches),
|
|
46
|
+
tree: readTree(record.tree)
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
@@ -90,6 +91,23 @@ function readEntries(value) {
|
|
|
90
91
|
}
|
|
91
92
|
return entries;
|
|
92
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* The tree entry out of a stored record, or none when malformed.
|
|
96
|
+
*
|
|
97
|
+
* @param value - The `tree` field as read from disk.
|
|
98
|
+
* @returns The entry, or undefined.
|
|
99
|
+
*/
|
|
100
|
+
function readTree(value) {
|
|
101
|
+
const record = asRecord(value);
|
|
102
|
+
if (!record)
|
|
103
|
+
return undefined;
|
|
104
|
+
return {
|
|
105
|
+
sha: typeof record.sha === 'string' ? record.sha : undefined,
|
|
106
|
+
sentAt: typeof record.sentAt === 'number' ? record.sentAt : undefined,
|
|
107
|
+
sentFor: typeof record.sentFor === 'string' ? record.sentFor : undefined,
|
|
108
|
+
triedAt: typeof record.triedAt === 'number' ? record.triedAt : undefined
|
|
109
|
+
};
|
|
110
|
+
}
|
|
93
111
|
/**
|
|
94
112
|
* The state of a repository nothing has been sent for.
|
|
95
113
|
*
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { RepoTreeEvent } from '../events/types/repo-tree.types.js';
|
|
2
|
+
import type { BuildTreeInput } from './types/snapshot.types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Build the record one default-branch tree becomes.
|
|
5
|
+
*
|
|
6
|
+
* The id is the repository and what is sent — each path with its names,
|
|
7
|
+
* sorted — and nothing else, not the agent: a retry, or a second developer or
|
|
8
|
+
* agent sending the same trunk, is the same record, and the gateway
|
|
9
|
+
* deduplicates it. Core identifies a tree the same way; no git oid is sent.
|
|
10
|
+
*
|
|
11
|
+
* @param input - Identity and the tree.
|
|
12
|
+
* @returns The tree event, within the contract's caps.
|
|
13
|
+
*/
|
|
14
|
+
export declare function buildRepoTree(input: BuildTreeInput): RepoTreeEvent;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { deriveEventId, sha256Hex } from '../events/event-id.js';
|
|
2
|
+
import { EVENT_SCHEMA_VERSION } from '../events/constants/events.constants.js';
|
|
3
|
+
import { TREE_EVENT_ID_PROVIDER, TREE_MAX_EVENT_BYTES, TREE_MAX_FILES, TREE_MAX_NAME_LENGTH, TREE_MAX_NAMES_PER_FILE } from '../git/constants/tree.constants.js';
|
|
4
|
+
import { compact } from '../core/object.js';
|
|
5
|
+
import { sendable } from './snapshot-event.js';
|
|
6
|
+
/**
|
|
7
|
+
* Build the record one default-branch tree becomes.
|
|
8
|
+
*
|
|
9
|
+
* The id is the repository and what is sent — each path with its names,
|
|
10
|
+
* sorted — and nothing else, not the agent: a retry, or a second developer or
|
|
11
|
+
* agent sending the same trunk, is the same record, and the gateway
|
|
12
|
+
* deduplicates it. Core identifies a tree the same way; no git oid is sent.
|
|
13
|
+
*
|
|
14
|
+
* @param input - Identity and the tree.
|
|
15
|
+
* @returns The tree event, within the contract's caps.
|
|
16
|
+
*/
|
|
17
|
+
export function buildRepoTree(input) {
|
|
18
|
+
const { identity, tree } = input;
|
|
19
|
+
const capped = capFiles(tree.files);
|
|
20
|
+
return compact({
|
|
21
|
+
schemaVersion: EVENT_SCHEMA_VERSION,
|
|
22
|
+
id: deriveEventId({
|
|
23
|
+
provider: TREE_EVENT_ID_PROVIDER,
|
|
24
|
+
providerEventType: 'repo.tree',
|
|
25
|
+
payloadFingerprint: contentFingerprint(identity.repository, capped.files)
|
|
26
|
+
}),
|
|
27
|
+
timestamp: identity.capturedAt,
|
|
28
|
+
event: { type: 'repo.tree', providerEventType: 'repo.tree' },
|
|
29
|
+
agent: { provider: identity.provider, name: identity.agentName },
|
|
30
|
+
session: { id: identity.sessionId, providerId: identity.sessionId },
|
|
31
|
+
developer: identity.installationId ? { installationId: identity.installationId } : undefined,
|
|
32
|
+
provider: identity.provider,
|
|
33
|
+
surface: identity.surface,
|
|
34
|
+
repository: identity.repository,
|
|
35
|
+
developer_id: identity.developerId,
|
|
36
|
+
captured_at: identity.capturedAt,
|
|
37
|
+
truncated: capped.truncated || tree.partial,
|
|
38
|
+
files: capped.files
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The files one event carries, in `ls-tree` order, within the schema's caps.
|
|
43
|
+
*
|
|
44
|
+
* Over a cap the contract says stop adding and mark the tree truncated: core
|
|
45
|
+
* still names it, and a name it never saw costs less than a 413, which loses
|
|
46
|
+
* the whole tree. A withheld path (credential-named, over-long) is not a
|
|
47
|
+
* truncation — it would never be sent.
|
|
48
|
+
*
|
|
49
|
+
* @param files - The tree's source files.
|
|
50
|
+
* @returns The files to send and whether any were cut.
|
|
51
|
+
*/
|
|
52
|
+
function capFiles(files) {
|
|
53
|
+
const kept = [];
|
|
54
|
+
let bytes = 0;
|
|
55
|
+
let truncated = false;
|
|
56
|
+
for (const file of files) {
|
|
57
|
+
if (!sendable(file.path))
|
|
58
|
+
continue;
|
|
59
|
+
const names = capNames(file.names);
|
|
60
|
+
const entry = { path: file.path, names: names.names };
|
|
61
|
+
const size = Buffer.byteLength(JSON.stringify(entry)) + 1;
|
|
62
|
+
if (kept.length === TREE_MAX_FILES || bytes + size > TREE_MAX_EVENT_BYTES)
|
|
63
|
+
return { files: kept, truncated: true };
|
|
64
|
+
truncated ||= names.truncated;
|
|
65
|
+
bytes += size;
|
|
66
|
+
kept.push(entry);
|
|
67
|
+
}
|
|
68
|
+
return { files: kept, truncated };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* One file's names within the schema's caps: names over the length dropped, the first 64 kept.
|
|
72
|
+
*
|
|
73
|
+
* @param names - Declaration names in first-seen order.
|
|
74
|
+
* @returns The names to send and whether any were dropped.
|
|
75
|
+
*/
|
|
76
|
+
function capNames(names) {
|
|
77
|
+
const fitting = names.filter((name) => name.length <= TREE_MAX_NAME_LENGTH);
|
|
78
|
+
const kept = fitting.slice(0, TREE_MAX_NAMES_PER_FILE);
|
|
79
|
+
return { names: kept, truncated: kept.length < names.length };
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* A digest of what one tree event says: the repository and every `path:names` line, sorted.
|
|
83
|
+
*
|
|
84
|
+
* @param repository - Sanitized repository identity.
|
|
85
|
+
* @param files - The files the event carries.
|
|
86
|
+
* @returns Hex digest.
|
|
87
|
+
*/
|
|
88
|
+
function contentFingerprint(repository, files) {
|
|
89
|
+
const lines = files.map((file) => `${file.path}:${file.names.join(',')}`).sort();
|
|
90
|
+
return sha256Hex([repository, ...lines].join('\n'));
|
|
91
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { FlowResult } from '../core/types/core.types.js';
|
|
2
|
+
import type { TreeFlowState, TreePipelineInput } from './types/snapshot.types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Send the default branch's source paths and declaration names, once per repository and identity.
|
|
5
|
+
*
|
|
6
|
+
* Shares the snapshot's deadline and never outlives it: an overrun stops
|
|
7
|
+
* without recording anything, so the next turn tries again. Never throws.
|
|
8
|
+
*
|
|
9
|
+
* @param pipeline - The snapshot flow's input, the state cache and the offline queue.
|
|
10
|
+
* @returns The final state and where the flow stopped.
|
|
11
|
+
*/
|
|
12
|
+
export declare function runTreePipeline(pipeline: TreePipelineInput): Promise<FlowResult<TreeFlowState>>;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { debugLog } from '../core/logger.js';
|
|
2
|
+
import { next, runFlow, step, stop } from '../core/pipe.js';
|
|
3
|
+
import { resolveDefaultBranch } from '../git/repo-snapshot.js';
|
|
4
|
+
import { collectRepoTree, resolveTreeSha } from '../git/repo-tree.js';
|
|
5
|
+
import { sanitizeValue } from '../privacy/sanitizer.js';
|
|
6
|
+
import { budgetedRunner, remainingMs, withinBudget } from './budget.js';
|
|
7
|
+
import { TREE_RETRY_INTERVAL_MS } from './constants/snapshot.constants.js';
|
|
8
|
+
import { STAGE_ENQUEUE, STAGE_SANITIZE, STAGE_TREE_COLLECT, STAGE_TREE_DUE, STAGE_TREE_RESOLVE, STOP_BUDGET_SPENT, STOP_NO_EVENT, STOP_TREE_NOT_DUE, STOP_TREE_NO_TRUNK, STOP_TREE_SENT, STOP_TREE_UNLISTED } from './constants/snapshot-stages.constants.js';
|
|
9
|
+
import { emptyState } from './snapshot-state.js';
|
|
10
|
+
import { buildRepoTree } from './tree-event.js';
|
|
11
|
+
/**
|
|
12
|
+
* The tree flow, as the list of stages it is.
|
|
13
|
+
*
|
|
14
|
+
* Cheapest first: the stored state costs no git at all, and once the tree has
|
|
15
|
+
* been sent it is the only stage that ever runs again.
|
|
16
|
+
*/
|
|
17
|
+
const TREE_STAGES = [
|
|
18
|
+
step(STAGE_TREE_DUE, due),
|
|
19
|
+
step(STAGE_TREE_RESOLVE, resolve),
|
|
20
|
+
step(STAGE_TREE_COLLECT, collect),
|
|
21
|
+
step(STAGE_SANITIZE, sanitize),
|
|
22
|
+
step(STAGE_ENQUEUE, enqueue)
|
|
23
|
+
];
|
|
24
|
+
/**
|
|
25
|
+
* Send the default branch's source paths and declaration names, once per repository and identity.
|
|
26
|
+
*
|
|
27
|
+
* Shares the snapshot's deadline and never outlives it: an overrun stops
|
|
28
|
+
* without recording anything, so the next turn tries again. Never throws.
|
|
29
|
+
*
|
|
30
|
+
* @param pipeline - The snapshot flow's input, the state cache and the offline queue.
|
|
31
|
+
* @returns The final state and where the flow stopped.
|
|
32
|
+
*/
|
|
33
|
+
export function runTreePipeline(pipeline) {
|
|
34
|
+
const initial = {
|
|
35
|
+
input: pipeline.input,
|
|
36
|
+
store: pipeline.store,
|
|
37
|
+
queue: pipeline.queue,
|
|
38
|
+
run: budgetedRunner(pipeline.input.run, pipeline.input.deadline)
|
|
39
|
+
};
|
|
40
|
+
return runFlow(TREE_STAGES, initial, (trace) => {
|
|
41
|
+
if (trace.outcome === 'next')
|
|
42
|
+
return;
|
|
43
|
+
debugLog(`tree flow ${trace.outcome} at ${trace.step}${trace.reason ? `: ${trace.reason}` : ''}`);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Whether this repository still owes its one tree.
|
|
48
|
+
*
|
|
49
|
+
* Once per repository per identity: after the first tree is queued nothing
|
|
50
|
+
* here runs again, so an ordinary turn pays one small state read and no git
|
|
51
|
+
* process. The files a session changes arrive in its turn summaries and branch
|
|
52
|
+
* snapshots anyway. A new identity — a re-setup with a new credential, a
|
|
53
|
+
* different account — owes the tree again. An attempt that failed waits out
|
|
54
|
+
* {@link TREE_RETRY_INTERVAL_MS}, so a repository the budget cannot list costs
|
|
55
|
+
* its second once an hour, not on every turn.
|
|
56
|
+
*
|
|
57
|
+
* @param state - Current flow state.
|
|
58
|
+
* @returns The state with what was stored, or a stop when nothing is owed yet.
|
|
59
|
+
*/
|
|
60
|
+
async function due(state) {
|
|
61
|
+
const stored = await state.store.read(state.input.repository);
|
|
62
|
+
const tree = stored.tree;
|
|
63
|
+
if (tree?.sentAt !== undefined && tree.sentFor === state.input.identity)
|
|
64
|
+
return stop({ ...state, stored }, STOP_TREE_SENT);
|
|
65
|
+
if (tree?.triedAt !== undefined && Date.parse(state.input.capturedAt) - tree.triedAt < TREE_RETRY_INTERVAL_MS)
|
|
66
|
+
return stop({ ...state, stored }, STOP_TREE_NOT_DUE);
|
|
67
|
+
return next({ ...state, stored });
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The trunk and its tree oid.
|
|
71
|
+
*
|
|
72
|
+
* @param state - Current flow state.
|
|
73
|
+
* @returns The state with the trunk and the tree, or a stop when there is none.
|
|
74
|
+
*/
|
|
75
|
+
async function resolve(state) {
|
|
76
|
+
const { cwd } = state.input;
|
|
77
|
+
const defaultBranch = await resolveDefaultBranch(cwd, state.run);
|
|
78
|
+
const treeSha = defaultBranch ? await resolveTreeSha(cwd, defaultBranch, state.run) : undefined;
|
|
79
|
+
if (!defaultBranch || !treeSha)
|
|
80
|
+
return stop(await triedNow(state), STOP_TREE_NO_TRUNK);
|
|
81
|
+
return next({ ...state, treeSha });
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* List the tree and search it for declaration names.
|
|
85
|
+
*
|
|
86
|
+
* The budget is checked after, not only before: a search the budget cut short
|
|
87
|
+
* returns nothing, which reads exactly like a tree with no declarations, and
|
|
88
|
+
* sending that would record the tree as named when it was not.
|
|
89
|
+
*
|
|
90
|
+
* ponytail: a tree that never fits in what the branch snapshot leaves of one
|
|
91
|
+
* second is never sent (agent-watch-core, 1,704 source files, takes ~200 ms).
|
|
92
|
+
* Upgrade: give the tree its own budget on its hourly attempt, or search it
|
|
93
|
+
* in slices across turns.
|
|
94
|
+
*
|
|
95
|
+
* @param state - Current flow state.
|
|
96
|
+
* @returns The state with the tree, or a stop when it could not be listed in time.
|
|
97
|
+
*/
|
|
98
|
+
async function collect(state) {
|
|
99
|
+
const tree = await collectRepoTree(state.input.cwd, state.treeSha, state.run);
|
|
100
|
+
if (remainingMs(state.input.deadline) === 0)
|
|
101
|
+
return stop(await triedNow(state), STOP_BUDGET_SPENT);
|
|
102
|
+
if (!tree)
|
|
103
|
+
return stop(await triedNow(state), STOP_TREE_UNLISTED);
|
|
104
|
+
return next({ ...state, tree });
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Record a failed attempt, so the next one waits out the retry interval.
|
|
108
|
+
*
|
|
109
|
+
* Written whatever is left of the deadline — usually nothing, which is why
|
|
110
|
+
* the attempt failed — because it is one small local file, and skipping it
|
|
111
|
+
* would put the same failing listing on every following turn.
|
|
112
|
+
*
|
|
113
|
+
* @param state - Current flow state.
|
|
114
|
+
* @returns The state, unchanged.
|
|
115
|
+
*/
|
|
116
|
+
async function triedNow(state) {
|
|
117
|
+
const stored = state.stored ?? emptyState();
|
|
118
|
+
await state.store.write(state.input.repository, { ...stored, tree: { ...stored.tree, triedAt: Date.parse(state.input.capturedAt) } }).catch(() => undefined);
|
|
119
|
+
return state;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Build the event and put it through the same scrubber every record gets.
|
|
123
|
+
*
|
|
124
|
+
* @param state - Current flow state.
|
|
125
|
+
* @returns The state carrying the event to send.
|
|
126
|
+
*/
|
|
127
|
+
async function sanitize(state) {
|
|
128
|
+
const event = buildRepoTree({ identity: state.input, tree: state.tree });
|
|
129
|
+
return next({ ...state, event: sanitizeValue(event) });
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Queue the event, then record the tree as sent — never the other way round.
|
|
133
|
+
* The tree oid stays in this local marker; it is not sent.
|
|
134
|
+
*
|
|
135
|
+
* Queued is sent: the queue retries until it is delivered. ponytail: a tree
|
|
136
|
+
* the queue gives up on (attempts or age exhausted) is not offered again for
|
|
137
|
+
* this identity; clearing `tree.sentAt` when the queue drops a `repo.tree`
|
|
138
|
+
* is the upgrade if that ever bites.
|
|
139
|
+
*
|
|
140
|
+
* @param state - Current flow state.
|
|
141
|
+
* @returns The state, unchanged.
|
|
142
|
+
*/
|
|
143
|
+
async function enqueue(state) {
|
|
144
|
+
if (!state.event)
|
|
145
|
+
return stop(state, STOP_NO_EVENT);
|
|
146
|
+
const { deadline, repository, capturedAt } = state.input;
|
|
147
|
+
const queued = await withinBudget(state.queue.enqueue([state.event]), deadline);
|
|
148
|
+
if (!queued)
|
|
149
|
+
return stop(await triedNow(state), STOP_BUDGET_SPENT);
|
|
150
|
+
const stored = state.stored ?? emptyState();
|
|
151
|
+
await withinBudget(state.store.write(repository, { ...stored, tree: { sha: state.treeSha, sentAt: Date.parse(capturedAt), sentFor: state.input.identity } }), deadline);
|
|
152
|
+
return next(state);
|
|
153
|
+
}
|