deepline 0.3.195 → 0.3.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +13 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +18 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +10 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +145 -29
- package/dist/bundling-sources/shared_libs/plays/docflow-binding-owner.ts +138 -14
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +57 -3
- package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +3 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +21 -0
- package/dist/bundling-sources/shared_libs/plays/workspace-bundle.ts +651 -0
- package/dist/cli/index.js +48 -2
- package/dist/cli/index.mjs +48 -2
- package/dist/{compiler-manifest-4bYusxBr.d.mts → compiler-manifest-Dpd2hEet.d.mts} +28 -0
- package/dist/{compiler-manifest-4bYusxBr.d.ts → compiler-manifest-Dpd2hEet.d.ts} +28 -0
- package/dist/index.d.mts +27 -2
- package/dist/index.d.ts +27 -2
- package/dist/index.js +7 -1
- package/dist/index.mjs +7 -1
- package/dist/install-integrity.json +3 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +24 -3
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,651 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A portable snapshot of what a workspace LOOKS like.
|
|
3
|
+
*
|
|
4
|
+
* The dashboard has exactly one seeded shape today — the hand-written arrays in
|
|
5
|
+
* `apps/deepline-api/src/lib/auth/local-dev-fixtures.ts` — and reviewing a UI
|
|
6
|
+
* against five invented plays tells you very little about how it holds up on a
|
|
7
|
+
* real one. A bundle is those same arrays, captured from a live workspace
|
|
8
|
+
* instead of typed by hand, so a preview or a local stack can be filled with
|
|
9
|
+
* data that has real shapes: real column counts, real name lengths, real
|
|
10
|
+
* provider mixes, real run durations, real failure modes.
|
|
11
|
+
*
|
|
12
|
+
* The bundle deliberately carries NO credentials and NO identifiers that bind
|
|
13
|
+
* it back to the workspace it came from. Everything under
|
|
14
|
+
* `integrationCredentials`, `oauthConnections`, `apiKeys`, `playSecrets` and
|
|
15
|
+
* friends is simply not part of the capture surface — not excluded by a filter
|
|
16
|
+
* that could regress, but absent from the types below, so there is no field for
|
|
17
|
+
* a secret to travel in. See `REDACTED_VALUE_KINDS` for what is rewritten on
|
|
18
|
+
* the way out.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { PlaySheetContract, PlayStaticPipeline } from './static-pipeline';
|
|
22
|
+
|
|
23
|
+
export const WORKSPACE_BUNDLE_API_VERSION =
|
|
24
|
+
'deepline.dev/workspace-bundle/v1' as const;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Wire-compatible with `LocalDevPlayFixture`. The loader feeds these straight
|
|
28
|
+
* into `internal.plays.registerOrgPlayRevisionInternal`, so the field names are
|
|
29
|
+
* that mutation's, not ours.
|
|
30
|
+
*/
|
|
31
|
+
export type BundlePlay = {
|
|
32
|
+
name: string;
|
|
33
|
+
description: string;
|
|
34
|
+
sourceCode: string;
|
|
35
|
+
sourceHash: string;
|
|
36
|
+
artifactStorageKey: string;
|
|
37
|
+
staticPipeline: PlayStaticPipeline;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** Wire-compatible with `LocalDevPlayRunFixture`. */
|
|
41
|
+
export type BundleRun = {
|
|
42
|
+
key: string;
|
|
43
|
+
playName: string;
|
|
44
|
+
/**
|
|
45
|
+
* Age at capture, not an absolute timestamp. A bundle loaded three weeks
|
|
46
|
+
* after it was captured should still read as "ran 20 minutes ago", because
|
|
47
|
+
* the feed's whole job is showing recency — absolute timestamps would make
|
|
48
|
+
* every loaded workspace look abandoned.
|
|
49
|
+
*/
|
|
50
|
+
ageMs: number;
|
|
51
|
+
status: 'completed' | 'failed';
|
|
52
|
+
inputSummary: Record<string, unknown>;
|
|
53
|
+
result?: Record<string, unknown>;
|
|
54
|
+
resultSummary?: Record<string, unknown>;
|
|
55
|
+
tableNamespace?: string;
|
|
56
|
+
error?: string;
|
|
57
|
+
logs: string[];
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/** Wire-compatible with `LocalDevPlaySheetFixture`. */
|
|
61
|
+
export type BundleSheet = {
|
|
62
|
+
playName: string;
|
|
63
|
+
tableNamespace: string;
|
|
64
|
+
runKey: string;
|
|
65
|
+
sheetContract: PlaySheetContract;
|
|
66
|
+
rows: Array<Record<string, unknown> & { __deeplineRowKey: string }>;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Wire-compatible with `createLocalDevMonitorFixtures`' output MINUS `orgId`,
|
|
71
|
+
* `installedAt` and `updatedAt`: those are minted by the loader against the
|
|
72
|
+
* destination org, and carrying the source org id is exactly the binding this
|
|
73
|
+
* format exists to avoid.
|
|
74
|
+
*/
|
|
75
|
+
export type BundleMonitor = {
|
|
76
|
+
monitorKey: string;
|
|
77
|
+
name: string;
|
|
78
|
+
sourceProvider: string;
|
|
79
|
+
sourceCapability: string;
|
|
80
|
+
tool: string;
|
|
81
|
+
payload: Record<string, unknown>;
|
|
82
|
+
controls: Record<string, unknown>;
|
|
83
|
+
outputs: Record<string, unknown>;
|
|
84
|
+
providerBinding: Record<string, unknown>;
|
|
85
|
+
billingState: Record<string, unknown>;
|
|
86
|
+
subjectKind: string;
|
|
87
|
+
subjectValue: string;
|
|
88
|
+
subjectLabel: string;
|
|
89
|
+
signalKey: string;
|
|
90
|
+
signalLabel: string;
|
|
91
|
+
signalGroupKey: string;
|
|
92
|
+
signalGroupLabel: string;
|
|
93
|
+
searchText: string;
|
|
94
|
+
publicPreview: Record<string, unknown>;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** Wire-compatible with `LocalDevMonitorEventFixture`. */
|
|
98
|
+
export type BundleMonitorEvent = {
|
|
99
|
+
monitorKey: string;
|
|
100
|
+
tableName: string;
|
|
101
|
+
payload: Record<string, unknown>;
|
|
102
|
+
/** Age at capture, for the same reason `BundleRun.ageMs` is. */
|
|
103
|
+
ageMs: number;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
export type WorkspaceBundleRedactionMode = 'pseudonymized' | 'raw';
|
|
107
|
+
|
|
108
|
+
export type WorkspaceBundle = {
|
|
109
|
+
apiVersion: typeof WORKSPACE_BUNDLE_API_VERSION;
|
|
110
|
+
capturedAt: string;
|
|
111
|
+
/**
|
|
112
|
+
* Enough to know what you are looking at, and nothing that resolves back to
|
|
113
|
+
* a tenant. A label the operator typed, never the org id or slug.
|
|
114
|
+
*/
|
|
115
|
+
source: { label: string };
|
|
116
|
+
redaction: {
|
|
117
|
+
mode: WorkspaceBundleRedactionMode;
|
|
118
|
+
/** Which value kinds were rewritten. Empty iff mode is 'raw'. */
|
|
119
|
+
kinds: readonly RedactedValueKind[];
|
|
120
|
+
};
|
|
121
|
+
plays: BundlePlay[];
|
|
122
|
+
runs: BundleRun[];
|
|
123
|
+
sheets: BundleSheet[];
|
|
124
|
+
monitors: BundleMonitor[];
|
|
125
|
+
monitorEvents: BundleMonitorEvent[];
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
export const REDACTED_VALUE_KINDS = [
|
|
129
|
+
'email',
|
|
130
|
+
'phone',
|
|
131
|
+
'linkedin',
|
|
132
|
+
'person_name',
|
|
133
|
+
'person_given',
|
|
134
|
+
'person_family',
|
|
135
|
+
] as const;
|
|
136
|
+
|
|
137
|
+
export type RedactedValueKind = (typeof REDACTED_VALUE_KINDS)[number];
|
|
138
|
+
|
|
139
|
+
const EMAIL_RE = /^[^\s@]+@[^\s@.]+\.[^\s@]+$/;
|
|
140
|
+
const LINKEDIN_RE = /^https?:\/\/([a-z0-9-]+\.)?linkedin\.com\/in\/[^\s]+$/i;
|
|
141
|
+
// Deliberately narrow: 7+ digits with optional punctuation and country code.
|
|
142
|
+
// A loose pattern here eats revenue figures and employee counts, which are the
|
|
143
|
+
// numbers that make a captured sheet look like real data.
|
|
144
|
+
const PHONE_RE = /^\+?[\d][\d\s().-]{6,}\d$/;
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Columns holding ONE token of a person's name.
|
|
148
|
+
*
|
|
149
|
+
* These are why the first capture of a real workspace leaked: `first_name:
|
|
150
|
+
* "Steven"` is a single capitalised word, so a rule demanding two or three
|
|
151
|
+
* words passed it straight through while correctly rewriting the `full_name`
|
|
152
|
+
* sitting beside it.
|
|
153
|
+
*/
|
|
154
|
+
const GIVEN_NAME_COLUMN_RE = /(^|_)(first|given|fore)_?name($|_)/i;
|
|
155
|
+
const FAMILY_NAME_COLUMN_RE = /(^|_)(last|family|sur)_?name($|_)/i;
|
|
156
|
+
|
|
157
|
+
/** Columns holding a whole person name. */
|
|
158
|
+
const PERSON_NAME_COLUMN_RE =
|
|
159
|
+
/(^|_)(full|contact|person|owner|founder|exec|prospect|lead|attendee)(_?name)?($|_)/i;
|
|
160
|
+
|
|
161
|
+
/** A corporate suffix settles it: an organisation, whatever the column says. */
|
|
162
|
+
const COMPANY_SUFFIX_RE =
|
|
163
|
+
/\b(inc|llc|ltd|limited|gmbh|corp|corporation|co|company|plc|holdings|group|labs|partners|ventures|capital|associates|solutions|services|systems|technologies)\b\.?/i;
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The same two things, found INSIDE free text rather than as a whole value.
|
|
167
|
+
*
|
|
168
|
+
* Run logs, result summaries and Play source code are prose and code, not
|
|
169
|
+
* cells — a real capture leaked `steven.wilhelm@…` out of a log line and
|
|
170
|
+
* `linkedin.com/in/dion-mokhtari` out of a result blob, because a whole-value
|
|
171
|
+
* matcher never looks inside a sentence. Only emails and LinkedIn URLs are
|
|
172
|
+
* swept this way: they are unambiguous. Scanning free text for phone numbers
|
|
173
|
+
* would eat request ids, row counts and timestamps.
|
|
174
|
+
*/
|
|
175
|
+
const EMAIL_IN_TEXT = /[\w.+-]+@[\w-]+(?:\.[\w-]+)+/g;
|
|
176
|
+
const LINKEDIN_IN_TEXT =
|
|
177
|
+
/https?:\/\/(?:[a-z0-9-]+\.)?linkedin\.com\/in\/[\w%-]+/gi;
|
|
178
|
+
|
|
179
|
+
/** One capitalised name token, tolerating apostrophes, hyphens and initials. */
|
|
180
|
+
const NAME_TOKEN = String.raw`\p{Lu}[\p{L}'’-]*\.?`;
|
|
181
|
+
const SINGLE_NAME_RE = new RegExp(`^${NAME_TOKEN}$`, 'u');
|
|
182
|
+
const FULL_NAME_RE = new RegExp(`^${NAME_TOKEN}(?: ${NAME_TOKEN}){1,3}$`, 'u');
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Deterministic within one capture, meaningless across captures.
|
|
186
|
+
*
|
|
187
|
+
* The salt is generated per capture and never written to the bundle, so the
|
|
188
|
+
* same input maps to the same pseudonym throughout one snapshot — joins,
|
|
189
|
+
* dedupes and group-bys keep working, and the UI keeps looking like it is
|
|
190
|
+
* showing a coherent dataset — while nothing in the published artifact lets you
|
|
191
|
+
* walk a pseudonym back to the person.
|
|
192
|
+
*/
|
|
193
|
+
export type Pseudonymizer = (value: string, kind: RedactedValueKind) => string;
|
|
194
|
+
|
|
195
|
+
const FIRST_NAMES = [
|
|
196
|
+
'Avery',
|
|
197
|
+
'Rowan',
|
|
198
|
+
'Marlowe',
|
|
199
|
+
'Sasha',
|
|
200
|
+
'Imani',
|
|
201
|
+
'Dara',
|
|
202
|
+
'Niko',
|
|
203
|
+
'Priya',
|
|
204
|
+
'Elena',
|
|
205
|
+
'Tobias',
|
|
206
|
+
'Naomi',
|
|
207
|
+
'Kwame',
|
|
208
|
+
'Lucia',
|
|
209
|
+
'Soren',
|
|
210
|
+
'Mira',
|
|
211
|
+
'Idris',
|
|
212
|
+
'Freya',
|
|
213
|
+
'Omar',
|
|
214
|
+
'Talia',
|
|
215
|
+
'Bastien',
|
|
216
|
+
'Ines',
|
|
217
|
+
'Hugo',
|
|
218
|
+
'Zola',
|
|
219
|
+
'Rafael',
|
|
220
|
+
];
|
|
221
|
+
|
|
222
|
+
const LAST_NAMES = [
|
|
223
|
+
'Okonkwo',
|
|
224
|
+
'Vance',
|
|
225
|
+
'Halloran',
|
|
226
|
+
'Moreau',
|
|
227
|
+
'Castellanos',
|
|
228
|
+
'Nakamura',
|
|
229
|
+
'Lindqvist',
|
|
230
|
+
'Abadi',
|
|
231
|
+
'Ferreira',
|
|
232
|
+
'Whitlock',
|
|
233
|
+
'Draper',
|
|
234
|
+
'Sorensen',
|
|
235
|
+
'Bergström',
|
|
236
|
+
'Nakashima',
|
|
237
|
+
'Okafor',
|
|
238
|
+
'Delacroix',
|
|
239
|
+
'Ramirez',
|
|
240
|
+
'Thorne',
|
|
241
|
+
'Adeyemi',
|
|
242
|
+
'Kowalski',
|
|
243
|
+
'Mensah',
|
|
244
|
+
'Beaumont',
|
|
245
|
+
'Ivanova',
|
|
246
|
+
'Quintero',
|
|
247
|
+
];
|
|
248
|
+
|
|
249
|
+
/** FNV-1a. Not a security primitive — it is salted upstream, see `createPseudonymizer`. */
|
|
250
|
+
function hash32(input: string): number {
|
|
251
|
+
let h = 0x811c9dc5;
|
|
252
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
253
|
+
h ^= input.charCodeAt(i);
|
|
254
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
255
|
+
}
|
|
256
|
+
return h >>> 0;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function pick<T>(list: readonly T[], seed: number): T {
|
|
260
|
+
return list[seed % list.length]!;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Split an identifier that encodes a person's name into its name tokens.
|
|
265
|
+
*
|
|
266
|
+
* Handles the three shapes that carry one in practice: an email local part
|
|
267
|
+
* (`steven.wilhelm`), a LinkedIn slug (`steven-wilhelm-4821`), and a plain
|
|
268
|
+
* name. Trailing disambiguation digits are dropped so `steven-wilhelm-4821`
|
|
269
|
+
* and `steven-wilhelm` resolve to the same person.
|
|
270
|
+
*/
|
|
271
|
+
function nameTokensOf(raw: string): string[] {
|
|
272
|
+
const tokens = raw
|
|
273
|
+
.split(/[\s._+-]+/)
|
|
274
|
+
.filter((token) => token !== '' && !/^\d+$/.test(token));
|
|
275
|
+
return tokens.length > 0 ? tokens : [raw];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Build a pseudonymizer bound to one random salt.
|
|
280
|
+
*
|
|
281
|
+
* Pass the salt explicitly only from tests. Production callers let it default
|
|
282
|
+
* so that two captures of the same workspace produce different pseudonyms and
|
|
283
|
+
* cannot be correlated with each other.
|
|
284
|
+
*/
|
|
285
|
+
export function createPseudonymizer(
|
|
286
|
+
salt: string = Math.random().toString(36).slice(2) +
|
|
287
|
+
Math.random().toString(36).slice(2),
|
|
288
|
+
): Pseudonymizer {
|
|
289
|
+
const cache = new Map<string, string>();
|
|
290
|
+
|
|
291
|
+
return (value, kind) => {
|
|
292
|
+
const cacheKey = `${kind}${value}`;
|
|
293
|
+
const cached = cache.get(cacheKey);
|
|
294
|
+
if (cached !== undefined) return cached;
|
|
295
|
+
|
|
296
|
+
const seed = hash32(`${salt}${kind}${value.toLowerCase()}`);
|
|
297
|
+
// Name tokens map per TOKEN, not per whole value, so `first_name`,
|
|
298
|
+
// `last_name` and `full_name` in one row still compose into a single
|
|
299
|
+
// consistent person rather than three unrelated ones.
|
|
300
|
+
const givenFor = (token: string) =>
|
|
301
|
+
pick(FIRST_NAMES, hash32(`${salt} person_given ${token.toLowerCase()}`));
|
|
302
|
+
const familyFor = (token: string) =>
|
|
303
|
+
pick(LAST_NAMES, hash32(`${salt} person_family ${token.toLowerCase()}`));
|
|
304
|
+
|
|
305
|
+
const tokens = value.trim().split(/\s+/);
|
|
306
|
+
const first = givenFor(tokens[0] ?? value);
|
|
307
|
+
const last = familyFor(tokens[tokens.length - 1] ?? value);
|
|
308
|
+
|
|
309
|
+
let replaced: string;
|
|
310
|
+
switch (kind) {
|
|
311
|
+
case 'email': {
|
|
312
|
+
// The domain survives. It is a company, not a person, and it is what
|
|
313
|
+
// makes a captured sheet still read as "twelve people at four
|
|
314
|
+
// companies" rather than as noise.
|
|
315
|
+
//
|
|
316
|
+
// The local part is rebuilt from ITS OWN name tokens, through the same
|
|
317
|
+
// per-token mapping the name columns use, so `steven.wilhelm@acme.com`
|
|
318
|
+
// lands on the same invented person as the `first_name` and
|
|
319
|
+
// `last_name` beside it. Otherwise a row reads as a mix-up rather than
|
|
320
|
+
// as a person, which is precisely what a reviewer notices.
|
|
321
|
+
const at = value.lastIndexOf('@');
|
|
322
|
+
const domain = value.slice(at + 1);
|
|
323
|
+
const localTokens = nameTokensOf(value.slice(0, at));
|
|
324
|
+
replaced = `${givenFor(localTokens[0]).toLowerCase()}.${familyFor(
|
|
325
|
+
localTokens[localTokens.length - 1],
|
|
326
|
+
).toLowerCase()}@${domain}`;
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
case 'linkedin': {
|
|
330
|
+
// Same reasoning as the email: the slug carries the person's name, so
|
|
331
|
+
// map its tokens rather than inventing an unrelated one.
|
|
332
|
+
const slug = value.replace(/\/+$/, '').split('/in/')[1] ?? value;
|
|
333
|
+
const slugTokens = nameTokensOf(slug);
|
|
334
|
+
replaced = `https://www.linkedin.com/in/${givenFor(
|
|
335
|
+
slugTokens[0],
|
|
336
|
+
).toLowerCase()}-${familyFor(
|
|
337
|
+
slugTokens[slugTokens.length - 1],
|
|
338
|
+
).toLowerCase()}-${(seed % 900000) + 100000}`;
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
case 'phone': {
|
|
342
|
+
// Keep the shape — length and punctuation drive column width, which is
|
|
343
|
+
// half of what a table review is actually looking at.
|
|
344
|
+
let digitIndex = 0;
|
|
345
|
+
const digits = String(1000000000 + (seed % 899999999));
|
|
346
|
+
replaced = value.replace(/\d/g, () => {
|
|
347
|
+
const next = digits[digitIndex % digits.length]!;
|
|
348
|
+
digitIndex += 1;
|
|
349
|
+
return next;
|
|
350
|
+
});
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
case 'person_name':
|
|
354
|
+
replaced = tokens.length === 1 ? first : `${first} ${last}`;
|
|
355
|
+
break;
|
|
356
|
+
case 'person_given':
|
|
357
|
+
replaced = first;
|
|
358
|
+
break;
|
|
359
|
+
case 'person_family':
|
|
360
|
+
replaced = last;
|
|
361
|
+
break;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
cache.set(cacheKey, replaced);
|
|
365
|
+
return replaced;
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Classify a single cell. Column name is a hint, not the decision: a column
|
|
371
|
+
* called `owner` holding `"Acme Holdings LLC"` is a company, and a column
|
|
372
|
+
* called `contact` holding an address is an email wherever it sits.
|
|
373
|
+
*/
|
|
374
|
+
export function classifyValue(
|
|
375
|
+
column: string,
|
|
376
|
+
value: unknown,
|
|
377
|
+
): RedactedValueKind | null {
|
|
378
|
+
if (typeof value !== 'string') return null;
|
|
379
|
+
const trimmed = value.trim();
|
|
380
|
+
if (trimmed === '') return null;
|
|
381
|
+
|
|
382
|
+
if (EMAIL_RE.test(trimmed)) return 'email';
|
|
383
|
+
if (LINKEDIN_RE.test(trimmed)) return 'linkedin';
|
|
384
|
+
if (PHONE_RE.test(trimmed) && /\d{7}/.test(trimmed.replace(/\D/g, ''))) {
|
|
385
|
+
return 'phone';
|
|
386
|
+
}
|
|
387
|
+
// A corporate suffix settles it before the column name gets a say: a value
|
|
388
|
+
// like "Mears Group, Inc." is an organisation wherever it sits.
|
|
389
|
+
if (COMPANY_SUFFIX_RE.test(trimmed)) return null;
|
|
390
|
+
|
|
391
|
+
// Single-token columns accept a single token; whole-name columns still
|
|
392
|
+
// require at least two, so a bare company in an `owner` column stays put.
|
|
393
|
+
if (GIVEN_NAME_COLUMN_RE.test(column) && SINGLE_NAME_RE.test(trimmed)) {
|
|
394
|
+
return 'person_given';
|
|
395
|
+
}
|
|
396
|
+
if (FAMILY_NAME_COLUMN_RE.test(column) && SINGLE_NAME_RE.test(trimmed)) {
|
|
397
|
+
return 'person_family';
|
|
398
|
+
}
|
|
399
|
+
if (PERSON_NAME_COLUMN_RE.test(column) && FULL_NAME_RE.test(trimmed)) {
|
|
400
|
+
return 'person_name';
|
|
401
|
+
}
|
|
402
|
+
return null;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Redact one value, however deeply it is buried.
|
|
407
|
+
*
|
|
408
|
+
* Play sheets routinely carry a JSON blob in a single cell — a provider's raw
|
|
409
|
+
* response, an evaluation record — and the first capture of a real workspace
|
|
410
|
+
* leaked a LinkedIn URL exactly that way: the cell was a *string*, so a
|
|
411
|
+
* top-level check saw one long opaque value and passed it through while the
|
|
412
|
+
* personal data sat one `JSON.parse` below it. Objects, arrays and
|
|
413
|
+
* JSON-in-a-string are all walked, and a JSON string is re-serialised so the
|
|
414
|
+
* cell keeps the shape the grid renders.
|
|
415
|
+
*/
|
|
416
|
+
function redactValue(
|
|
417
|
+
column: string,
|
|
418
|
+
value: unknown,
|
|
419
|
+
pseudonymize: Pseudonymizer,
|
|
420
|
+
): unknown {
|
|
421
|
+
if (Array.isArray(value)) {
|
|
422
|
+
return value.map((item) => redactValue(column, item, pseudonymize));
|
|
423
|
+
}
|
|
424
|
+
if (value && typeof value === 'object') {
|
|
425
|
+
return Object.fromEntries(
|
|
426
|
+
Object.entries(value as Record<string, unknown>).map(([key, nested]) => [
|
|
427
|
+
key,
|
|
428
|
+
redactValue(key, nested, pseudonymize),
|
|
429
|
+
]),
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
if (typeof value === 'string') {
|
|
433
|
+
const trimmed = value.trim();
|
|
434
|
+
if (
|
|
435
|
+
(trimmed.startsWith('{') && trimmed.endsWith('}')) ||
|
|
436
|
+
(trimmed.startsWith('[') && trimmed.endsWith(']'))
|
|
437
|
+
) {
|
|
438
|
+
try {
|
|
439
|
+
const parsed: unknown = JSON.parse(trimmed);
|
|
440
|
+
if (parsed && typeof parsed === 'object') {
|
|
441
|
+
return JSON.stringify(redactValue(column, parsed, pseudonymize));
|
|
442
|
+
}
|
|
443
|
+
} catch {
|
|
444
|
+
// Not JSON after all — a cell that merely starts with a brace. Fall
|
|
445
|
+
// through and treat it as the plain string it is.
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
const kind = classifyValue(column, value);
|
|
449
|
+
if (kind) return pseudonymize(value, kind);
|
|
450
|
+
// Not a personal value on its own — but it may still CONTAIN one. Log
|
|
451
|
+
// lines, result summaries and Play source all carry addresses inline.
|
|
452
|
+
return redactFreeText(value, pseudonymize);
|
|
453
|
+
}
|
|
454
|
+
return value;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** Rewrite every email and LinkedIn URL found inside a block of text. */
|
|
458
|
+
export function redactFreeText(
|
|
459
|
+
text: string,
|
|
460
|
+
pseudonymize: Pseudonymizer,
|
|
461
|
+
): string {
|
|
462
|
+
return text
|
|
463
|
+
.replace(EMAIL_IN_TEXT, (match) => pseudonymize(match, 'email'))
|
|
464
|
+
.replace(LINKEDIN_IN_TEXT, (match) => pseudonymize(match, 'linkedin'));
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
function redactRow(
|
|
468
|
+
row: Record<string, unknown>,
|
|
469
|
+
pseudonymize: Pseudonymizer,
|
|
470
|
+
): Record<string, unknown> {
|
|
471
|
+
return redactValue('', row, pseudonymize) as Record<string, unknown>;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Rewrite every personal value in a bundle, in place of the originals.
|
|
476
|
+
*
|
|
477
|
+
* This runs at CAPTURE time, before the bundle is ever written to disk or
|
|
478
|
+
* pushed to R2 — so the raw values exist only in the capturing process's
|
|
479
|
+
* memory. Redacting after the fact would mean a raw artifact had already
|
|
480
|
+
* existed somewhere, which is the thing worth avoiding.
|
|
481
|
+
*/
|
|
482
|
+
export function redactWorkspaceBundle(
|
|
483
|
+
bundle: WorkspaceBundle,
|
|
484
|
+
pseudonymize: Pseudonymizer = createPseudonymizer(),
|
|
485
|
+
): WorkspaceBundle {
|
|
486
|
+
const text = (value: string) => redactFreeText(value, pseudonymize);
|
|
487
|
+
const blob = <T>(value: T): T => redactValue('', value, pseudonymize) as T;
|
|
488
|
+
|
|
489
|
+
return {
|
|
490
|
+
...bundle,
|
|
491
|
+
redaction: { mode: 'pseudonymized', kinds: REDACTED_VALUE_KINDS },
|
|
492
|
+
// Plays: a description or a hardcoded literal in the source routinely
|
|
493
|
+
// names a real prospect. The source stays valid — an address is replaced
|
|
494
|
+
// by an address.
|
|
495
|
+
plays: bundle.plays.map((play) => ({
|
|
496
|
+
...play,
|
|
497
|
+
description: text(play.description),
|
|
498
|
+
sourceCode: text(play.sourceCode),
|
|
499
|
+
staticPipeline: blob(play.staticPipeline),
|
|
500
|
+
})),
|
|
501
|
+
// Runs: logs and result summaries are where a real capture leaked most.
|
|
502
|
+
// They are prose about the rows the Play touched, so they name them.
|
|
503
|
+
runs: bundle.runs.map((run) => ({
|
|
504
|
+
...run,
|
|
505
|
+
logs: run.logs.map(text),
|
|
506
|
+
inputSummary: blob(run.inputSummary),
|
|
507
|
+
...(run.result ? { result: blob(run.result) } : {}),
|
|
508
|
+
...(run.resultSummary ? { resultSummary: blob(run.resultSummary) } : {}),
|
|
509
|
+
...(run.error ? { error: text(run.error) } : {}),
|
|
510
|
+
})),
|
|
511
|
+
sheets: bundle.sheets.map((sheet) => ({
|
|
512
|
+
...sheet,
|
|
513
|
+
rows: sheet.rows.map((row) => {
|
|
514
|
+
const redacted = redactRow(row, pseudonymize);
|
|
515
|
+
// The row key is an identity, not a value, and the sheet's own
|
|
516
|
+
// contract keys off it. Rewriting it would break row updates.
|
|
517
|
+
redacted.__deeplineRowKey = row.__deeplineRowKey;
|
|
518
|
+
return redacted as BundleSheet['rows'][number];
|
|
519
|
+
}),
|
|
520
|
+
})),
|
|
521
|
+
monitorEvents: bundle.monitorEvents.map((event) => ({
|
|
522
|
+
...event,
|
|
523
|
+
payload: redactRow(event.payload, pseudonymize),
|
|
524
|
+
})),
|
|
525
|
+
monitors: bundle.monitors.map((monitor) => ({
|
|
526
|
+
...monitor,
|
|
527
|
+
payload: blob(monitor.payload),
|
|
528
|
+
outputs: blob(monitor.outputs),
|
|
529
|
+
subjectValue: text(monitor.subjectValue),
|
|
530
|
+
subjectLabel: text(monitor.subjectLabel),
|
|
531
|
+
// `searchText` is a flattened concatenation of the fields above it, so it
|
|
532
|
+
// leaks whatever they held. Recompute rather than redact.
|
|
533
|
+
searchText: [monitor.name, monitor.subjectLabel, monitor.signalLabel]
|
|
534
|
+
.filter(Boolean)
|
|
535
|
+
.join(' ')
|
|
536
|
+
.toLowerCase(),
|
|
537
|
+
publicPreview: redactRow(monitor.publicPreview, pseudonymize),
|
|
538
|
+
})),
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
export class WorkspaceBundleFormatError extends Error {}
|
|
543
|
+
|
|
544
|
+
function requireArray(value: unknown, field: string): unknown[] {
|
|
545
|
+
if (!Array.isArray(value)) {
|
|
546
|
+
throw new WorkspaceBundleFormatError(
|
|
547
|
+
`workspace bundle field "${field}" must be an array, got ${typeof value}`,
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
return value;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Validate an untrusted bundle.
|
|
555
|
+
*
|
|
556
|
+
* A bundle arrives from R2 and is fed to mutations that write a workspace, so
|
|
557
|
+
* it is untrusted input even when we wrote it ourselves — a stale object under
|
|
558
|
+
* a reused key is the ordinary case, not the exotic one. Fails loudly and
|
|
559
|
+
* names the field; never repairs, defaults, or drops a malformed record.
|
|
560
|
+
*/
|
|
561
|
+
export function parseWorkspaceBundle(input: unknown): WorkspaceBundle {
|
|
562
|
+
if (typeof input !== 'object' || input === null) {
|
|
563
|
+
throw new WorkspaceBundleFormatError(
|
|
564
|
+
`workspace bundle must be an object, got ${input === null ? 'null' : typeof input}`,
|
|
565
|
+
);
|
|
566
|
+
}
|
|
567
|
+
const raw = input as Record<string, unknown>;
|
|
568
|
+
|
|
569
|
+
if (raw.apiVersion !== WORKSPACE_BUNDLE_API_VERSION) {
|
|
570
|
+
throw new WorkspaceBundleFormatError(
|
|
571
|
+
`unsupported workspace bundle apiVersion ${JSON.stringify(raw.apiVersion)}; this build reads ${WORKSPACE_BUNDLE_API_VERSION}`,
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
const redaction = raw.redaction as WorkspaceBundle['redaction'] | undefined;
|
|
576
|
+
if (
|
|
577
|
+
!redaction ||
|
|
578
|
+
(redaction.mode !== 'pseudonymized' && redaction.mode !== 'raw')
|
|
579
|
+
) {
|
|
580
|
+
throw new WorkspaceBundleFormatError(
|
|
581
|
+
'workspace bundle is missing a redaction mode; refusing to load data whose handling is unstated',
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
const bundle: WorkspaceBundle = {
|
|
586
|
+
apiVersion: WORKSPACE_BUNDLE_API_VERSION,
|
|
587
|
+
capturedAt: typeof raw.capturedAt === 'string' ? raw.capturedAt : '',
|
|
588
|
+
source: { label: String((raw.source as { label?: unknown })?.label ?? '') },
|
|
589
|
+
redaction,
|
|
590
|
+
plays: requireArray(raw.plays, 'plays') as BundlePlay[],
|
|
591
|
+
runs: requireArray(raw.runs, 'runs') as BundleRun[],
|
|
592
|
+
sheets: requireArray(raw.sheets, 'sheets') as BundleSheet[],
|
|
593
|
+
monitors: requireArray(raw.monitors, 'monitors') as BundleMonitor[],
|
|
594
|
+
monitorEvents: requireArray(
|
|
595
|
+
raw.monitorEvents,
|
|
596
|
+
'monitorEvents',
|
|
597
|
+
) as BundleMonitorEvent[],
|
|
598
|
+
};
|
|
599
|
+
|
|
600
|
+
// Referential integrity: a run or sheet naming a play the bundle does not
|
|
601
|
+
// carry would seed a workspace whose feed rows open onto nothing.
|
|
602
|
+
const playNames = new Set(bundle.plays.map((play) => play.name));
|
|
603
|
+
for (const run of bundle.runs) {
|
|
604
|
+
if (!playNames.has(run.playName)) {
|
|
605
|
+
throw new WorkspaceBundleFormatError(
|
|
606
|
+
`run ${JSON.stringify(run.key)} names play ${JSON.stringify(run.playName)}, which the bundle does not carry`,
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
const runKeys = new Set(bundle.runs.map((run) => run.key));
|
|
611
|
+
for (const sheet of bundle.sheets) {
|
|
612
|
+
if (!playNames.has(sheet.playName)) {
|
|
613
|
+
throw new WorkspaceBundleFormatError(
|
|
614
|
+
`sheet for namespace ${JSON.stringify(sheet.tableNamespace)} names play ${JSON.stringify(sheet.playName)}, which the bundle does not carry`,
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
if (!runKeys.has(sheet.runKey)) {
|
|
618
|
+
throw new WorkspaceBundleFormatError(
|
|
619
|
+
`sheet for namespace ${JSON.stringify(sheet.tableNamespace)} names run ${JSON.stringify(sheet.runKey)}, which the bundle does not carry`,
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
const monitorKeys = new Set(
|
|
624
|
+
bundle.monitors.map((monitor) => monitor.monitorKey),
|
|
625
|
+
);
|
|
626
|
+
for (const event of bundle.monitorEvents) {
|
|
627
|
+
if (!monitorKeys.has(event.monitorKey)) {
|
|
628
|
+
throw new WorkspaceBundleFormatError(
|
|
629
|
+
`monitor event names monitor ${JSON.stringify(event.monitorKey)}, which the bundle does not carry`,
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
return bundle;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/** Counts for an operator to eyeball before pushing or loading. */
|
|
638
|
+
export function summarizeWorkspaceBundle(bundle: WorkspaceBundle): string {
|
|
639
|
+
const rows = bundle.sheets.reduce(
|
|
640
|
+
(total, sheet) => total + sheet.rows.length,
|
|
641
|
+
0,
|
|
642
|
+
);
|
|
643
|
+
return [
|
|
644
|
+
`${bundle.plays.length} plays`,
|
|
645
|
+
`${bundle.runs.length} runs`,
|
|
646
|
+
`${bundle.sheets.length} sheets / ${rows} rows`,
|
|
647
|
+
`${bundle.monitors.length} monitors`,
|
|
648
|
+
`${bundle.monitorEvents.length} monitor events`,
|
|
649
|
+
bundle.redaction.mode,
|
|
650
|
+
].join(' · ');
|
|
651
|
+
}
|