@shieldaai/agent-map 0.1.0-alpha.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +652 -272
- package/dist/assets.js +16 -26
- package/dist/boundary-flags.d.ts +73 -0
- package/dist/boundary-flags.js +352 -0
- package/dist/boundary.d.ts +18 -3
- package/dist/boundary.js +71 -21
- package/dist/cli-entry.d.ts +11 -0
- package/dist/cli-entry.js +48 -0
- package/dist/cli.js +40 -43
- package/dist/common.d.ts +34 -0
- package/dist/common.js +94 -0
- package/dist/http.js +78 -51
- package/dist/identity-cli.js +93 -46
- package/dist/identity-http.js +226 -94
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/mcp.js +40 -5
- package/dist/oidc.d.ts +5 -0
- package/dist/oidc.js +121 -38
- package/dist/policy-reader.d.ts +56 -3
- package/dist/policy-reader.js +823 -110
- package/dist/projection-fields.d.ts +4 -0
- package/dist/projection-fields.js +144 -0
- package/dist/service/changes.d.ts +5 -0
- package/dist/service/changes.js +255 -0
- package/dist/service/context.d.ts +33 -0
- package/dist/service/context.js +2 -0
- package/dist/service/discovery.d.ts +6 -0
- package/dist/service/discovery.js +31 -0
- package/dist/service/explain.d.ts +12 -0
- package/dist/service/explain.js +126 -0
- package/dist/service/flags.d.ts +20 -0
- package/dist/service/flags.js +70 -0
- package/dist/service/inventory.d.ts +4 -0
- package/dist/service/inventory.js +62 -0
- package/dist/service/json.d.ts +8 -0
- package/dist/service/json.js +7 -0
- package/dist/service/operations/discovery.d.ts +5 -0
- package/dist/service/operations/discovery.js +90 -0
- package/dist/service/operations/explain-access.d.ts +3 -0
- package/dist/service/operations/explain-access.js +25 -0
- package/dist/service/operations/find-agents-with-access.d.ts +3 -0
- package/dist/service/operations/find-agents-with-access.js +120 -0
- package/dist/service/operations/get-agent-access.d.ts +3 -0
- package/dist/service/operations/get-agent-access.js +75 -0
- package/dist/service/operations/get-agent-inventory.d.ts +3 -0
- package/dist/service/operations/get-agent-inventory.js +42 -0
- package/dist/service/operations/get-changes.d.ts +3 -0
- package/dist/service/operations/get-changes.js +11 -0
- package/dist/service/operations/get-inventory-health.d.ts +3 -0
- package/dist/service/operations/get-inventory-health.js +81 -0
- package/dist/service/operations/get-resource-access.d.ts +3 -0
- package/dist/service/operations/get-resource-access.js +48 -0
- package/dist/service/operations/list-agents.d.ts +3 -0
- package/dist/service/operations/list-agents.js +92 -0
- package/dist/service/operations/list-owners-resources.d.ts +5 -0
- package/dist/service/operations/list-owners-resources.js +69 -0
- package/dist/service/ordering.d.ts +9 -0
- package/dist/service/ordering.js +47 -0
- package/dist/service/outside-guard.d.ts +68 -0
- package/dist/service/outside-guard.js +313 -0
- package/dist/service/policy.d.ts +26 -0
- package/dist/service/policy.js +299 -0
- package/dist/service/schemas.d.ts +16 -0
- package/dist/service/schemas.js +192 -0
- package/dist/service/summaries.d.ts +32 -0
- package/dist/service/summaries.js +247 -0
- package/dist/service/work.d.ts +33 -0
- package/dist/service/work.js +116 -0
- package/dist/service.d.ts +5 -5
- package/dist/service.js +110 -645
- package/dist/store/ancestors.d.ts +9 -0
- package/dist/store/ancestors.js +86 -0
- package/dist/store/discovery.d.ts +8 -0
- package/dist/store/discovery.js +288 -0
- package/dist/store/evidence.d.ts +12 -0
- package/dist/store/evidence.js +90 -0
- package/dist/store/identity-scope.d.ts +9 -0
- package/dist/store/identity-scope.js +71 -0
- package/dist/store/nodes.d.ts +24 -0
- package/dist/store/nodes.js +237 -0
- package/dist/store/projection.d.ts +19 -0
- package/dist/store/projection.js +242 -0
- package/dist/store/queries.d.ts +48 -0
- package/dist/store/queries.js +302 -0
- package/dist/store/read.d.ts +81 -0
- package/dist/store/read.js +135 -0
- package/dist/store/selection.d.ts +3 -0
- package/dist/store/selection.js +102 -0
- package/dist/store.d.ts +15 -0
- package/dist/store.js +102 -395
- package/dist/transport.d.ts +42 -2
- package/dist/transport.js +144 -16
- package/dist/types.d.ts +184 -4
- package/dist/types.js +52 -11
- package/package.json +16 -3
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type MapSource } from '../types.js';
|
|
2
|
+
import { StoreRead } from './read.js';
|
|
3
|
+
/**
|
|
4
|
+
* Delegation ancestors last, after every source's own nodes and edges, so they never crowd out
|
|
5
|
+
* an agent's own evidence: each source's ancestors are added only if they fit what is left of
|
|
6
|
+
* the node, edge and byte budgets (keeping a quarter of the bytes for evidence). Otherwise none
|
|
7
|
+
* are added and the projection says so.
|
|
8
|
+
*/
|
|
9
|
+
export declare function loadAncestors(read: StoreRead, sources: MapSource[]): Promise<void>;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import neo4j from 'neo4j-driver';
|
|
2
|
+
import { MapQueryError } from '../types.js';
|
|
3
|
+
import { safeNode, text } from './nodes.js';
|
|
4
|
+
import { ANCESTOR_TASKS, ANCESTORS, EDGES } from './queries.js';
|
|
5
|
+
import { StoreRead } from './read.js';
|
|
6
|
+
/**
|
|
7
|
+
* Delegation ancestors last, after every source's own nodes and edges, so they never crowd out
|
|
8
|
+
* an agent's own evidence: each source's ancestors are added only if they fit what is left of
|
|
9
|
+
* the node, edge and byte budgets (keeping a quarter of the bytes for evidence). Otherwise none
|
|
10
|
+
* are added and the projection says so.
|
|
11
|
+
*/
|
|
12
|
+
export async function loadAncestors(read, sources) {
|
|
13
|
+
const { accessMode } = read;
|
|
14
|
+
for (const job of read.ancestorJobs) {
|
|
15
|
+
const index = sources.findIndex(item => item.id === job.sourceId);
|
|
16
|
+
const current = sources[index].current;
|
|
17
|
+
const loaded = new Set(current.nodes.map(node => node.id));
|
|
18
|
+
const entities = new Set(current.nodes.map(node => node.entityId));
|
|
19
|
+
const nodeRoom = read.limits.nodes - read.nodeCount;
|
|
20
|
+
const found = [];
|
|
21
|
+
const foundIds = new Set();
|
|
22
|
+
await read.run(ANCESTORS[accessMode], {
|
|
23
|
+
...job.params,
|
|
24
|
+
GuardDeploymentIds: current.nodes.filter(node => node.label === 'GuardDeployment').map(node => node.id),
|
|
25
|
+
nodeLimit: neo4j.int(read.limits.nodes + 1),
|
|
26
|
+
}, item => {
|
|
27
|
+
const ancestor = text(item.get('id'));
|
|
28
|
+
if (!loaded.has(ancestor) && !foundIds.has(ancestor) && found.length <= nodeRoom) {
|
|
29
|
+
foundIds.add(ancestor);
|
|
30
|
+
found.push(ancestor);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
let notLoaded = found.length > nodeRoom;
|
|
34
|
+
const records = [], extra = [];
|
|
35
|
+
if (!notLoaded && found.length) {
|
|
36
|
+
const requested = new Set(found);
|
|
37
|
+
await read.run(ANCESTOR_TASKS, { ...job.params, ancestorTaskIds: found, nodeLimit: neo4j.int(found.length + 1) }, item => {
|
|
38
|
+
if (item.get('label') !== 'GuardTask')
|
|
39
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
40
|
+
const node = safeNode(item.get('node'), 'GuardTask', job.sourceId, current.id);
|
|
41
|
+
// Each requested id once: a second node with the same id is malformed.
|
|
42
|
+
if (!requested.delete(node.id) || entities.has(node.entityId))
|
|
43
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
44
|
+
entities.add(node.entityId);
|
|
45
|
+
records.push(node);
|
|
46
|
+
});
|
|
47
|
+
// Only delegation and handoff edges touch ancestors (their grants and deployments
|
|
48
|
+
// are not read).
|
|
49
|
+
const ancestorSet = new Set(records.map(node => node.id));
|
|
50
|
+
const taskIds = [
|
|
51
|
+
...current.nodes.filter(node => node.label === 'GuardTask').map(node => node.id),
|
|
52
|
+
...ancestorSet,
|
|
53
|
+
];
|
|
54
|
+
const tasks = new Set(taskIds);
|
|
55
|
+
const edgeRoom = read.limits.edges - read.edgeCount;
|
|
56
|
+
await read.run(EDGES, {
|
|
57
|
+
...job.params,
|
|
58
|
+
nodeIds: taskIds,
|
|
59
|
+
relationships: ['G_DELEGATED_FROM', 'G_DERIVED_FROM'],
|
|
60
|
+
edgeLimit: neo4j.int(read.limits.edges + 1),
|
|
61
|
+
}, item => {
|
|
62
|
+
const source = text(item.get('source')), target = text(item.get('target'));
|
|
63
|
+
const kind = item.get('kind');
|
|
64
|
+
if ((kind !== 'G_DELEGATED_FROM' && kind !== 'G_DERIVED_FROM') || !tasks.has(source) || !tasks.has(target))
|
|
65
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
66
|
+
if (!ancestorSet.has(source) && !ancestorSet.has(target))
|
|
67
|
+
return;
|
|
68
|
+
if (extra.length <= edgeRoom)
|
|
69
|
+
extra.push({ source, target, kind });
|
|
70
|
+
});
|
|
71
|
+
const bytes = Buffer.byteLength(JSON.stringify([records, extra]), 'utf8');
|
|
72
|
+
notLoaded = extra.length > edgeRoom || read.materializedBytes + bytes > read.limits.bytes * 0.75;
|
|
73
|
+
if (!notLoaded) {
|
|
74
|
+
read.charge([records, extra]);
|
|
75
|
+
read.nodeCount += records.length;
|
|
76
|
+
read.edgeCount += extra.length;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
sources[index] = {
|
|
80
|
+
...sources[index],
|
|
81
|
+
current: notLoaded
|
|
82
|
+
? { ...current, ancestorsNotLoaded: true }
|
|
83
|
+
: { ...current, nodes: [...current.nodes, ...records], edges: [...current.edges, ...extra] },
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type MapView } from '../types.js';
|
|
2
|
+
import { StoreRead } from './read.js';
|
|
3
|
+
/**
|
|
4
|
+
* Discovery reads load no nodes: fixed, bounded queries per source, then the same publication
|
|
5
|
+
* recheck as every other read. `undeclared` counts (and pages) activity no declared agent
|
|
6
|
+
* accounts for; `unclaimedIdentities` lists IAM roles and users no declared binding names.
|
|
7
|
+
*/
|
|
8
|
+
export declare function readDiscovery(read: StoreRead): Promise<MapView>;
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import neo4j from 'neo4j-driver';
|
|
2
|
+
import { immutableJson } from '@shieldaai/agent-guard/internal';
|
|
3
|
+
import { collectionKey } from '../policy-reader.js';
|
|
4
|
+
import { MAP_LABELS, MapQueryError, } from '../types.js';
|
|
5
|
+
import { converted, coreCaptureTimes, coverage, text } from './nodes.js';
|
|
6
|
+
import { AWS_COLLECTION_COLUMNS, AWS_COLLECTION_LIMIT, AWS_COLLECTIONS, AWS_MEMBER_LIMIT, BINDINGS_WITHOUT_IDENTITY, CLAIM_LIMIT, CLAIMED_IDENTITIES, UNCLAIMED_COUNTS, UNCLAIMED_PAGE, UNDECLARED_COUNT_LIMIT, UNDECLARED_COUNTS, UNDECLARED_PRINCIPALS, UNDECLARED_TASKS, } from './queries.js';
|
|
7
|
+
import { StoreRead } from './read.js';
|
|
8
|
+
/**
|
|
9
|
+
* Discovery reads load no nodes: fixed, bounded queries per source, then the same publication
|
|
10
|
+
* recheck as every other read. `undeclared` counts (and pages) activity no declared agent
|
|
11
|
+
* accounts for; `unclaimedIdentities` lists IAM roles and users no declared binding names.
|
|
12
|
+
*/
|
|
13
|
+
export async function readDiscovery(read) {
|
|
14
|
+
const { scope, selection } = read;
|
|
15
|
+
const malformed = () => {
|
|
16
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
17
|
+
};
|
|
18
|
+
const whole = (value, maximum) => {
|
|
19
|
+
const number = converted(value);
|
|
20
|
+
return typeof number === 'number' && Number.isSafeInteger(number) && number >= 0 && number <= maximum
|
|
21
|
+
? number
|
|
22
|
+
: malformed();
|
|
23
|
+
};
|
|
24
|
+
const flag = (value) => value === null || value === undefined ? null : typeof value === 'boolean' ? value : malformed();
|
|
25
|
+
const page = selection.page;
|
|
26
|
+
const listed = [];
|
|
27
|
+
const claimed = new Set();
|
|
28
|
+
let claimsExceeded = false;
|
|
29
|
+
let bindingsWithoutIdentity = false;
|
|
30
|
+
// The strictest enrolled cloud age per tenant, under the host ceiling.
|
|
31
|
+
const maximumAges = new Map();
|
|
32
|
+
for (const record of read.selected) {
|
|
33
|
+
const sourceId = text(record.get('id'));
|
|
34
|
+
const tenantId = text(record.get('tenantId'));
|
|
35
|
+
const currentId = text(record.get('currentId'));
|
|
36
|
+
const profile = record.get('profile');
|
|
37
|
+
if (!scope.tenantIds.includes(tenantId) || !['core', 'host', 'host-mcp'].includes(profile))
|
|
38
|
+
malformed();
|
|
39
|
+
const params = { ...read.common, sourceId, tenantId, projectionId: currentId, labels: [...MAP_LABELS] };
|
|
40
|
+
const captures = coreCaptureTimes(record.get('currentCoverage'));
|
|
41
|
+
let undeclared;
|
|
42
|
+
if (selection.undeclared === true) {
|
|
43
|
+
const counts = await read.run(UNDECLARED_COUNTS, {
|
|
44
|
+
...params,
|
|
45
|
+
countLimit: neo4j.int(UNDECLARED_COUNT_LIMIT + 1),
|
|
46
|
+
});
|
|
47
|
+
if (counts.length !== 1)
|
|
48
|
+
malformed();
|
|
49
|
+
const tasks = whole(counts[0].get('tasks'), UNDECLARED_COUNT_LIMIT + 1);
|
|
50
|
+
const principals = whole(counts[0].get('principals'), UNDECLARED_COUNT_LIMIT + 1);
|
|
51
|
+
let listing;
|
|
52
|
+
if (page !== undefined) {
|
|
53
|
+
const paging = { ...params, pageOffset: neo4j.int(page.offset), pageLimit: neo4j.int(page.limit) };
|
|
54
|
+
const seen = new Set();
|
|
55
|
+
// A second row for one entity is malformed, never a longer list.
|
|
56
|
+
const once = (id) => {
|
|
57
|
+
if (seen.has(id))
|
|
58
|
+
malformed();
|
|
59
|
+
seen.add(id);
|
|
60
|
+
return id;
|
|
61
|
+
};
|
|
62
|
+
if (page.kind === 'tasks') {
|
|
63
|
+
const items = [];
|
|
64
|
+
await read.run(UNDECLARED_TASKS, paging, item => {
|
|
65
|
+
if (items.length >= page.limit)
|
|
66
|
+
malformed();
|
|
67
|
+
const tools = converted(item.get('tools'));
|
|
68
|
+
if (!Array.isArray(tools) || tools.length > 33)
|
|
69
|
+
malformed();
|
|
70
|
+
const names = [...new Set(tools.map(tool => text(tool)))].sort();
|
|
71
|
+
const principalId = item.get('principalId');
|
|
72
|
+
const parentDeclared = item.get('parentDeclared');
|
|
73
|
+
if (typeof parentDeclared !== 'boolean')
|
|
74
|
+
malformed();
|
|
75
|
+
const row = {
|
|
76
|
+
id: once(text(item.get('id'))),
|
|
77
|
+
taskId: text(item.get('taskId')),
|
|
78
|
+
principalId: principalId === null || principalId === undefined ? null : text(principalId),
|
|
79
|
+
delegated: flag(item.get('delegated')),
|
|
80
|
+
derived: flag(item.get('derived')),
|
|
81
|
+
revoked: flag(item.get('revoked')),
|
|
82
|
+
parentDeclared,
|
|
83
|
+
tools: names.slice(0, 32),
|
|
84
|
+
toolsTruncated: tools.length > 32,
|
|
85
|
+
};
|
|
86
|
+
read.charge(row);
|
|
87
|
+
items.push(row);
|
|
88
|
+
});
|
|
89
|
+
listing = { kind: 'tasks', items };
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
const items = [];
|
|
93
|
+
await read.run(UNDECLARED_PRINCIPALS, paging, item => {
|
|
94
|
+
if (items.length >= page.limit)
|
|
95
|
+
malformed();
|
|
96
|
+
const tasks = whole(item.get('tasks'), 1001);
|
|
97
|
+
const row = {
|
|
98
|
+
id: once(text(item.get('id'))),
|
|
99
|
+
principalId: text(item.get('principalId')),
|
|
100
|
+
tasks: Math.min(tasks, 1000),
|
|
101
|
+
tasksCapped: tasks > 1000,
|
|
102
|
+
};
|
|
103
|
+
read.charge(row);
|
|
104
|
+
items.push(row);
|
|
105
|
+
});
|
|
106
|
+
listing = { kind: 'principals', items };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
undeclared = {
|
|
110
|
+
tasks: Math.min(tasks, UNDECLARED_COUNT_LIMIT),
|
|
111
|
+
tasksCapped: tasks > UNDECLARED_COUNT_LIMIT,
|
|
112
|
+
principals: Math.min(principals, UNDECLARED_COUNT_LIMIT),
|
|
113
|
+
principalsCapped: principals > UNDECLARED_COUNT_LIMIT,
|
|
114
|
+
...(listing ? { page: listing } : {}),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
const enrolled = record.get('policyCoverage');
|
|
119
|
+
if (enrolled !== null && enrolled !== undefined)
|
|
120
|
+
maximumAges.set(tenantId, Math.min(maximumAges.get(tenantId) ?? read.limits.policyAge, read.maximumAgeFor(enrolled)));
|
|
121
|
+
if (!claimsExceeded)
|
|
122
|
+
await read.run(CLAIMED_IDENTITIES, { ...params, claimLimit: neo4j.int(CLAIM_LIMIT - claimed.size + 1) }, item => {
|
|
123
|
+
const arn = item.get('arn');
|
|
124
|
+
if (typeof arn !== 'string' || !arn || arn.length > 2048)
|
|
125
|
+
malformed();
|
|
126
|
+
claimed.add(arn);
|
|
127
|
+
if (claimed.size > CLAIM_LIMIT)
|
|
128
|
+
claimsExceeded = true;
|
|
129
|
+
});
|
|
130
|
+
if (!bindingsWithoutIdentity) {
|
|
131
|
+
const [open] = await read.run(BINDINGS_WITHOUT_IDENTITY, params);
|
|
132
|
+
// Only an explicit "none" clears it.
|
|
133
|
+
bindingsWithoutIdentity = open?.get('open') !== false;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
listed.push({
|
|
137
|
+
id: sourceId,
|
|
138
|
+
tenantId,
|
|
139
|
+
sourceId: text(record.get('nativeSourceId')),
|
|
140
|
+
profile,
|
|
141
|
+
current: {
|
|
142
|
+
id: currentId,
|
|
143
|
+
snapshotId: text(record.get('currentSnapshotId')),
|
|
144
|
+
generatedAt: text(record.get('currentGeneratedAt')),
|
|
145
|
+
status: 'published',
|
|
146
|
+
nodes: [],
|
|
147
|
+
edges: [],
|
|
148
|
+
// Only what this read rests on: when the tasks were captured.
|
|
149
|
+
coverage: { tasks: { status: 'complete', ...(captures.tasks ? { observedAt: captures.tasks } : {}) } },
|
|
150
|
+
...(undeclared ? { undeclared } : {}),
|
|
151
|
+
},
|
|
152
|
+
previous: null,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
let unclaimedIdentities;
|
|
156
|
+
if (selection.unclaimedIdentities === true) {
|
|
157
|
+
const readCollections = async () => (await read.run(AWS_COLLECTIONS, { ...read.common, collectionLimit: neo4j.int(AWS_COLLECTION_LIMIT + 1) })).map(item => Object.fromEntries(AWS_COLLECTION_COLUMNS.map(column => [column, converted(item.get(column))])));
|
|
158
|
+
const collections = await readCollections();
|
|
159
|
+
if (collections.length > AWS_COLLECTION_LIMIT)
|
|
160
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
161
|
+
const stamp = read.now();
|
|
162
|
+
const claims = [...claimed].sort();
|
|
163
|
+
const accounts = [];
|
|
164
|
+
const complete = new Map();
|
|
165
|
+
let total = 0;
|
|
166
|
+
for (const collection of collections) {
|
|
167
|
+
const { tenantId, partition, accountId, capturedAt } = collection;
|
|
168
|
+
// Without these the account cannot even be named.
|
|
169
|
+
if (typeof tenantId !== 'string' ||
|
|
170
|
+
!scope.tenantIds.includes(tenantId) ||
|
|
171
|
+
typeof partition !== 'string' ||
|
|
172
|
+
!['aws', 'aws-cn', 'aws-us-gov'].includes(partition) ||
|
|
173
|
+
typeof accountId !== 'string' ||
|
|
174
|
+
!/^[0-9]{12}$/.test(accountId)) {
|
|
175
|
+
malformed();
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
const account = (reason, observedAt, counts = null) => ({
|
|
179
|
+
tenantId,
|
|
180
|
+
partition,
|
|
181
|
+
accountId,
|
|
182
|
+
status: reason === null && counts ? 'complete' : 'unknown',
|
|
183
|
+
reason,
|
|
184
|
+
observedAt,
|
|
185
|
+
observed: counts?.observed ?? null,
|
|
186
|
+
unclaimed: counts?.unclaimed ?? null,
|
|
187
|
+
});
|
|
188
|
+
const id = collection.id;
|
|
189
|
+
if (typeof id !== 'string' ||
|
|
190
|
+
id !== collectionKey(scope.companyId, tenantId, partition, accountId) ||
|
|
191
|
+
complete.has(id) ||
|
|
192
|
+
collection.schemaVersion !== 1 ||
|
|
193
|
+
collection.status !== 'complete' ||
|
|
194
|
+
typeof collection.contentHash !== 'string' ||
|
|
195
|
+
!/^[a-f0-9]{64}$/.test(collection.contentHash) ||
|
|
196
|
+
typeof capturedAt !== 'number' ||
|
|
197
|
+
!Number.isSafeInteger(capturedAt) ||
|
|
198
|
+
capturedAt < 0 ||
|
|
199
|
+
capturedAt > stamp) {
|
|
200
|
+
accounts.push(account('collection_invalid', null));
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
const observedAt = new Date(capturedAt * 1000).toISOString();
|
|
204
|
+
if (stamp - capturedAt > (maximumAges.get(tenantId) ?? read.limits.policyAge)) {
|
|
205
|
+
accounts.push(account('stale', observedAt));
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
// Without the full set of claims, "unclaimed" would overstate.
|
|
209
|
+
if (claimsExceeded) {
|
|
210
|
+
accounts.push(account('claims_limit_exceeded', observedAt));
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
const counted = await read.run(UNCLAIMED_COUNTS, {
|
|
214
|
+
...read.common,
|
|
215
|
+
collectionId: id,
|
|
216
|
+
collectionTenantId: tenantId,
|
|
217
|
+
claimed: claims,
|
|
218
|
+
memberLimit: neo4j.int(AWS_MEMBER_LIMIT + 1),
|
|
219
|
+
});
|
|
220
|
+
if (counted.length !== 1)
|
|
221
|
+
malformed();
|
|
222
|
+
const observed = whole(counted[0].get('observed'), AWS_MEMBER_LIMIT + 1);
|
|
223
|
+
const unclaimed = whole(counted[0].get('unclaimed') ?? 0, AWS_MEMBER_LIMIT + 1);
|
|
224
|
+
const inconsistent = whole(counted[0].get('inconsistent') ?? 0, AWS_MEMBER_LIMIT + 1);
|
|
225
|
+
if (observed > AWS_MEMBER_LIMIT)
|
|
226
|
+
accounts.push(account('limit_exceeded', observedAt));
|
|
227
|
+
else if (inconsistent)
|
|
228
|
+
accounts.push(account('changed', observedAt));
|
|
229
|
+
else {
|
|
230
|
+
const judged = account(null, observedAt, { observed, unclaimed });
|
|
231
|
+
accounts.push(judged);
|
|
232
|
+
complete.set(id, judged);
|
|
233
|
+
total += unclaimed;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const identities = [];
|
|
237
|
+
if (page !== undefined && complete.size && page.offset < total)
|
|
238
|
+
await read.run(UNCLAIMED_PAGE, {
|
|
239
|
+
...read.common,
|
|
240
|
+
collectionIds: [...complete.keys()],
|
|
241
|
+
claimed: claims,
|
|
242
|
+
pageOffset: neo4j.int(page.offset),
|
|
243
|
+
pageLimit: neo4j.int(page.limit),
|
|
244
|
+
}, item => {
|
|
245
|
+
const account = complete.get(item.get('collectionId'));
|
|
246
|
+
const arn = item.get('arn'), name = item.get('name'), path = item.get('path');
|
|
247
|
+
const role = item.get('role'), user = item.get('user');
|
|
248
|
+
// The same shape the collector accepted: one kind, and an ARN built from its parts.
|
|
249
|
+
if (identities.length >= page.limit ||
|
|
250
|
+
!account ||
|
|
251
|
+
typeof role !== 'boolean' ||
|
|
252
|
+
typeof user !== 'boolean' ||
|
|
253
|
+
role === user ||
|
|
254
|
+
typeof name !== 'string' ||
|
|
255
|
+
!/^[A-Za-z0-9+=,.@_-]{1,128}$/.test(name) ||
|
|
256
|
+
typeof path !== 'string' ||
|
|
257
|
+
path.length > 512 ||
|
|
258
|
+
!/^\/(?:[\x21-\x7e]+\/)?$/.test(path) ||
|
|
259
|
+
arn !== `arn:${account.partition}:iam::${account.accountId}:${role ? 'role' : 'user'}${path}${name}` ||
|
|
260
|
+
claimed.has(arn)) {
|
|
261
|
+
malformed();
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const row = {
|
|
265
|
+
arn,
|
|
266
|
+
kind: role ? 'role' : 'user',
|
|
267
|
+
name,
|
|
268
|
+
path,
|
|
269
|
+
tenantId: account.tenantId,
|
|
270
|
+
partition: account.partition,
|
|
271
|
+
accountId: account.accountId,
|
|
272
|
+
};
|
|
273
|
+
read.charge(row);
|
|
274
|
+
identities.push(row);
|
|
275
|
+
});
|
|
276
|
+
// A collection published during the read would mix two captures.
|
|
277
|
+
if (JSON.stringify(collections) !== JSON.stringify(await readCollections()))
|
|
278
|
+
malformed();
|
|
279
|
+
unclaimedIdentities = { accounts, identities, total, bindingsWithoutIdentity };
|
|
280
|
+
}
|
|
281
|
+
await read.recheckSources();
|
|
282
|
+
try {
|
|
283
|
+
return immutableJson({ sources: listed, ...(unclaimedIdentities ? { unclaimedIdentities } : {}) }, 16, 200_000, 4 * 1024 * 1024);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
287
|
+
}
|
|
288
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type IdentityScopePlan, type PolicyPlan } from '../policy-reader.js';
|
|
2
|
+
import { type JsonObject } from '../types.js';
|
|
3
|
+
import { StoreRead } from './read.js';
|
|
4
|
+
/** Identity-scope rows for one plan, each bounded; a row that is too large stands in as truncated. */
|
|
5
|
+
export declare function readScopeRows(read: StoreRead, plan: IdentityScopePlan, params: Record<string, unknown>, chargeRows: boolean): Promise<JsonObject[]>;
|
|
6
|
+
/** Policy rows for one plan, bounded in count and bytes. */
|
|
7
|
+
export declare function readPolicyRows(read: StoreRead, plan: PolicyPlan, params: Record<string, unknown>, chargeRows: boolean): Promise<JsonObject[]>;
|
|
8
|
+
/**
|
|
9
|
+
* Reads every piece of evidence a second time, after the publication recheck: if a row, a
|
|
10
|
+
* classification or an identity-scope record changed while the read ran, the read is refused.
|
|
11
|
+
*/
|
|
12
|
+
export declare function verifyEvidence(read: StoreRead): Promise<void>;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import neo4j from 'neo4j-driver';
|
|
2
|
+
import { IDENTITY_SCOPE_COLUMNS, IDENTITY_SCOPE_QUERY, identityScopeEvidence, identityScopeParameters, POLICY_COLUMNS, POLICY_QUERY, policyEvidence, policyParameters, safeIdentityScopeRow, safePolicyRow, } from '../policy-reader.js';
|
|
3
|
+
import { MapQueryError } from '../types.js';
|
|
4
|
+
import { boundedRecords, converted, oversized, sortedBySerialization } from './nodes.js';
|
|
5
|
+
import { checked, scoped } from './queries.js';
|
|
6
|
+
import { StoreRead } from './read.js';
|
|
7
|
+
/** Identity-scope rows for one plan, each bounded; a row that is too large stands in as truncated. */
|
|
8
|
+
export async function readScopeRows(read, plan, params, chargeRows) {
|
|
9
|
+
if (!plan.probes.length)
|
|
10
|
+
return [];
|
|
11
|
+
const rows = [];
|
|
12
|
+
let readBytes = 0;
|
|
13
|
+
await read.run(IDENTITY_SCOPE_QUERY, {
|
|
14
|
+
...params,
|
|
15
|
+
scopePairs: identityScopeParameters(plan),
|
|
16
|
+
candidateLimit: neo4j.int(plan.candidateLimit),
|
|
17
|
+
buckets: plan.buckets,
|
|
18
|
+
// A malformed fan-out repeats a probe's row; the classifier marks that one record
|
|
19
|
+
// invalid, so only a much larger blow-up fails the whole read.
|
|
20
|
+
scopeRowLimit: neo4j.int(plan.probes.length * 8 + 1),
|
|
21
|
+
}, item => {
|
|
22
|
+
if (rows.length >= plan.probes.length * 8)
|
|
23
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
24
|
+
const raw = Object.fromEntries(IDENTITY_SCOPE_COLUMNS.map(column => [column, converted(item.get(column))]));
|
|
25
|
+
// An identity whose row is too large, or arrives after the read's byte budget, is
|
|
26
|
+
// reported truncated for that identity only; the rest of the read stands.
|
|
27
|
+
let safe;
|
|
28
|
+
try {
|
|
29
|
+
safe = safeIdentityScopeRow(raw);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
safe = oversized(raw.key);
|
|
33
|
+
}
|
|
34
|
+
const bytes = Buffer.byteLength(JSON.stringify(safe), 'utf8');
|
|
35
|
+
if (readBytes + bytes > 4 * 1024 * 1024)
|
|
36
|
+
safe = oversized(raw.key);
|
|
37
|
+
else
|
|
38
|
+
readBytes += bytes;
|
|
39
|
+
if (chargeRows)
|
|
40
|
+
read.charge(safe);
|
|
41
|
+
rows.push(safe);
|
|
42
|
+
});
|
|
43
|
+
return sortedBySerialization(rows);
|
|
44
|
+
}
|
|
45
|
+
/** Policy rows for one plan, bounded in count and bytes. */
|
|
46
|
+
export async function readPolicyRows(read, plan, params, chargeRows) {
|
|
47
|
+
if (!plan.probes.length)
|
|
48
|
+
return [];
|
|
49
|
+
const rows = [];
|
|
50
|
+
let readBytes = 0;
|
|
51
|
+
const pairs = policyParameters(plan);
|
|
52
|
+
if (Buffer.byteLength(JSON.stringify(pairs), 'utf8') > 4 * 1024 * 1024)
|
|
53
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
54
|
+
await read.run(POLICY_QUERY, { ...params, policyPairs: pairs, policyRowLimit: neo4j.int(plan.probes.length * 3 + 1) }, item => {
|
|
55
|
+
if (rows.length >= plan.probes.length * 3)
|
|
56
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
57
|
+
const raw = Object.fromEntries(POLICY_COLUMNS.map(column => [column, converted(item.get(column))]));
|
|
58
|
+
const safe = safePolicyRow(raw);
|
|
59
|
+
readBytes += Buffer.byteLength(JSON.stringify(safe), 'utf8');
|
|
60
|
+
if (readBytes > 4 * 1024 * 1024)
|
|
61
|
+
throw new MapQueryError('MAP_LIMIT_EXCEEDED');
|
|
62
|
+
if (chargeRows)
|
|
63
|
+
read.charge(safe);
|
|
64
|
+
rows.push(safe);
|
|
65
|
+
});
|
|
66
|
+
// Sort only the bounded materialized rows. A server ORDER BY would
|
|
67
|
+
// expand malformed relationship fan-out before applying its LIMIT.
|
|
68
|
+
return sortedBySerialization(rows);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Reads every piece of evidence a second time, after the publication recheck: if a row, a
|
|
72
|
+
* classification or an identity-scope record changed while the read ran, the read is refused.
|
|
73
|
+
*/
|
|
74
|
+
export async function verifyEvidence(read) {
|
|
75
|
+
for (const checked of read.policyChecks) {
|
|
76
|
+
const rows = await readPolicyRows(read, checked.plan, checked.params, false);
|
|
77
|
+
if (JSON.stringify(rows) !== JSON.stringify(checked.rows))
|
|
78
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
79
|
+
const refreshed = policyEvidence(checked.plan, rows, read.now(), checked.maximumAge);
|
|
80
|
+
if (JSON.stringify(refreshed) !== JSON.stringify(checked.projection.policyEvidence ?? []))
|
|
81
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
82
|
+
const scopeRows = await readScopeRows(read, checked.scopePlan, checked.params, false);
|
|
83
|
+
if (JSON.stringify(scopeRows) !== JSON.stringify(checked.scopeRows))
|
|
84
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
85
|
+
const scoped = boundedRecords(identityScopeEvidence(checked.scopePlan, scopeRows, read.now(), checked.maximumAge, checked.readable), checked.scopeBudget);
|
|
86
|
+
if (JSON.stringify(scoped) !==
|
|
87
|
+
JSON.stringify(checked.projection.identityScope ?? checked.projection.identityReach ?? []))
|
|
88
|
+
throw new MapQueryError('MAP_UNAVAILABLE');
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type MapSource } from '../types.js';
|
|
2
|
+
import { StoreRead } from './read.js';
|
|
3
|
+
/**
|
|
4
|
+
* Identity scope, once each projection holds its ancestors. A one-agent read lists every bucket
|
|
5
|
+
* per identity; an outside-guard read (list_resources, get_resource_access) asks only about the
|
|
6
|
+
* buckets visible bindings map to. It is optional evidence throughout: whatever cannot be
|
|
7
|
+
* planned, read or fitted is left out, and the read answers without it.
|
|
8
|
+
*/
|
|
9
|
+
export declare function loadIdentityScope(read: StoreRead, sources: MapSource[]): Promise<void>;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { identityScopeEvidence, identityScopePlan } from '../policy-reader.js';
|
|
2
|
+
import { MapQueryError } from '../types.js';
|
|
3
|
+
import { readScopeRows } from './evidence.js';
|
|
4
|
+
import { boundBuckets, boundedRecords, SCOPE_HEADROOM_BYTES, SCOPE_RECORD_BYTES } from './nodes.js';
|
|
5
|
+
import { StoreRead } from './read.js';
|
|
6
|
+
/**
|
|
7
|
+
* Identity scope, once each projection holds its ancestors. A one-agent read lists every bucket
|
|
8
|
+
* per identity; an outside-guard read (list_resources, get_resource_access) asks only about the
|
|
9
|
+
* buckets visible bindings map to. It is optional evidence throughout: whatever cannot be
|
|
10
|
+
* planned, read or fitted is left out, and the read answers without it.
|
|
11
|
+
*/
|
|
12
|
+
export async function loadIdentityScope(read, sources) {
|
|
13
|
+
const { scope } = read;
|
|
14
|
+
for (const job of read.scopeJobs) {
|
|
15
|
+
const index = sources.findIndex(item => item.id === job.sourceId);
|
|
16
|
+
const current = sources[index].current;
|
|
17
|
+
const buckets = boundBuckets(current.nodes);
|
|
18
|
+
// Identity scope is optional evidence: if planning it needs more work than allowed,
|
|
19
|
+
// the read goes on without it (records read unknown, reach cannot be none_found).
|
|
20
|
+
let scopePlan;
|
|
21
|
+
try {
|
|
22
|
+
scopePlan = !job.reach
|
|
23
|
+
? // Each distinct identity can return about 1,500 candidates; eight keeps a broad
|
|
24
|
+
// fleet inside the read's byte budget, and the rest report not_evaluated.
|
|
25
|
+
identityScopePlan(scope.companyId, sources[index].tenantId, current.nodes, current.edges, Math.max(0, 8 - read.scopePairCount), null, Date.parse(current.generatedAt))
|
|
26
|
+
: buckets.length
|
|
27
|
+
? identityScopePlan(scope.companyId, sources[index].tenantId, current.nodes, current.edges, Math.max(0, 32 - read.scopePairCount), buckets, Date.parse(current.generatedAt))
|
|
28
|
+
: identityScopePlan(scope.companyId, sources[index].tenantId, [], [], 0);
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
if (!(error instanceof MapQueryError) || error.code !== 'MAP_LIMIT_EXCEEDED')
|
|
32
|
+
throw error;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
read.scopePairCount += scopePlan.probes.length;
|
|
36
|
+
// Scope rows have their own byte bound in readScope; only the bounded records count
|
|
37
|
+
// toward the view, so a broad role cannot crowd out the rest. A scope-only read with
|
|
38
|
+
// an unreadable age bound, or a malformed row fan-out, also goes on without it.
|
|
39
|
+
let scopeRows;
|
|
40
|
+
try {
|
|
41
|
+
if (scopePlan.probes.length && !job.check.plan.probes.length)
|
|
42
|
+
job.check.maximumAge = read.maximumAgeFor(job.enrolled);
|
|
43
|
+
scopeRows = await readScopeRows(read, scopePlan, job.check.params, false);
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (!(error instanceof MapQueryError) ||
|
|
47
|
+
!['MAP_LIMIT_EXCEEDED', 'MAP_UNAVAILABLE'].includes(error.code) ||
|
|
48
|
+
!read.transactionOpen())
|
|
49
|
+
throw error;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
// The lists are optional detail: they get what is left of the read's byte budget,
|
|
53
|
+
// less headroom for the final copy. If even the bare records do not fit, the read
|
|
54
|
+
// answers without identity scope.
|
|
55
|
+
const scopeBudget = Math.max(0, Math.min(SCOPE_RECORD_BYTES, read.limits.bytes - SCOPE_HEADROOM_BYTES - read.materializedBytes));
|
|
56
|
+
const identityScope = boundedRecords(identityScopeEvidence(scopePlan, scopeRows, read.now(), job.check.maximumAge, job.check.readable), scopeBudget);
|
|
57
|
+
if (Buffer.byteLength(JSON.stringify(identityScope), 'utf8') > scopeBudget)
|
|
58
|
+
continue;
|
|
59
|
+
for (const item of identityScope)
|
|
60
|
+
read.charge(item);
|
|
61
|
+
const updated = {
|
|
62
|
+
...current,
|
|
63
|
+
...(job.reach ? { identityReach: identityScope } : { identityScope }),
|
|
64
|
+
};
|
|
65
|
+
sources[index] = { ...sources[index], current: updated };
|
|
66
|
+
job.check.scopePlan = scopePlan;
|
|
67
|
+
job.check.scopeRows = scopeRows;
|
|
68
|
+
job.check.scopeBudget = scopeBudget;
|
|
69
|
+
job.check.projection = updated;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type JsonObject, type MapCoverage, type MapIdentityScope, type MapNode } from '../types.js';
|
|
2
|
+
export declare const SCOPE_RECORD_BYTES: number;
|
|
3
|
+
export declare const SCOPE_HEADROOM_BYTES: number;
|
|
4
|
+
/**
|
|
5
|
+
* A stable order for bounded, already materialized rows. Each row is serialized once: the
|
|
6
|
+
* comparator would otherwise re-serialize rows of up to 1 MiB on every comparison.
|
|
7
|
+
*/
|
|
8
|
+
export declare function sortedBySerialization(rows: JsonObject[]): JsonObject[];
|
|
9
|
+
/**
|
|
10
|
+
* Tools sharing a role each list their own extras. Past the budget (1 MiB, or what is left of
|
|
11
|
+
* the read's byte budget if that is less), keep every record but drop the lists and report
|
|
12
|
+
* them truncated, rather than fail the read.
|
|
13
|
+
*/
|
|
14
|
+
export declare function boundedRecords(records: readonly MapIdentityScope[], budget?: number): readonly MapIdentityScope[];
|
|
15
|
+
/** Stands in for a scope row that could not be kept; the classifier reads it as truncated. */
|
|
16
|
+
export declare const oversized: (key: unknown) => JsonObject;
|
|
17
|
+
/** S3 bucket names that visible host bindings map to; the outside-guard reach asks only about these. */
|
|
18
|
+
export declare function boundBuckets(nodes: readonly MapNode[]): string[];
|
|
19
|
+
export declare function text(value: unknown): string;
|
|
20
|
+
export declare function converted(value: unknown): unknown;
|
|
21
|
+
export declare function safeNode(raw: unknown, label: unknown, sourceId: string, projectionId: string): MapNode;
|
|
22
|
+
export declare function observationTime(value: unknown): number | undefined;
|
|
23
|
+
export declare function coreCaptureTimes(raw: unknown): Partial<Record<'tasks' | 'tools', string>>;
|
|
24
|
+
export declare function coverage(nodes: readonly MapNode[], captures: Partial<Record<'tasks' | 'tools', string>>, profile: string, summary: boolean, now: number): MapCoverage;
|