@lobu/connector-worker 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -0
- package/dist/__tests__/redact.test.d.ts +2 -0
- package/dist/__tests__/redact.test.d.ts.map +1 -0
- package/dist/__tests__/redact.test.js +135 -0
- package/dist/__tests__/redact.test.js.map +1 -0
- package/dist/bin.d.ts +11 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +129 -0
- package/dist/bin.js.map +1 -0
- package/dist/daemon/client.d.ts +256 -0
- package/dist/daemon/client.d.ts.map +1 -0
- package/dist/daemon/client.js +152 -0
- package/dist/daemon/client.js.map +1 -0
- package/dist/daemon/executor.d.ts +25 -0
- package/dist/daemon/executor.d.ts.map +1 -0
- package/dist/daemon/executor.js +492 -0
- package/dist/daemon/executor.js.map +1 -0
- package/dist/daemon/index.d.ts +12 -0
- package/dist/daemon/index.d.ts.map +1 -0
- package/dist/daemon/index.js +9 -0
- package/dist/daemon/index.js.map +1 -0
- package/dist/daemon/worker.d.ts +54 -0
- package/dist/daemon/worker.d.ts.map +1 -0
- package/dist/daemon/worker.js +144 -0
- package/dist/daemon/worker.js.map +1 -0
- package/dist/embeddings.d.ts +9 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +126 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/executor/child-runner.d.ts +9 -0
- package/dist/executor/child-runner.d.ts.map +1 -0
- package/dist/executor/child-runner.js +317 -0
- package/dist/executor/child-runner.js.map +1 -0
- package/dist/executor/interface.d.ts +45 -0
- package/dist/executor/interface.d.ts.map +1 -0
- package/dist/executor/interface.js +2 -0
- package/dist/executor/interface.js.map +1 -0
- package/dist/executor/redact.d.ts +28 -0
- package/dist/executor/redact.d.ts.map +1 -0
- package/dist/executor/redact.js +108 -0
- package/dist/executor/redact.js.map +1 -0
- package/dist/executor/runtime.d.ts +45 -0
- package/dist/executor/runtime.d.ts.map +1 -0
- package/dist/executor/runtime.js +82 -0
- package/dist/executor/runtime.js.map +1 -0
- package/dist/executor/subprocess.d.ts +46 -0
- package/dist/executor/subprocess.d.ts.map +1 -0
- package/dist/executor/subprocess.js +378 -0
- package/dist/executor/subprocess.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/runtime-deps.d.ts +33 -0
- package/dist/runtime-deps.d.ts.map +1 -0
- package/dist/runtime-deps.js +48 -0
- package/dist/runtime-deps.js.map +1 -0
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +70 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Child Runner - Entry point for forked subprocess
|
|
3
|
+
*
|
|
4
|
+
* Receives compiled connector code + context via IPC,
|
|
5
|
+
* dynamically imports the ConnectorRuntime class, and executes sync()/execute().
|
|
6
|
+
* Streams normalized content chunks and checkpoint updates back via IPC.
|
|
7
|
+
*/
|
|
8
|
+
import { rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
import { pathToFileURL } from 'node:url';
|
|
11
|
+
import { normalizeEventEnvelope } from './runtime.js';
|
|
12
|
+
const CONTENT_CHUNK_SIZE = 100;
|
|
13
|
+
function stripInternalOptions(options) {
|
|
14
|
+
const result = {};
|
|
15
|
+
for (const [key, value] of Object.entries(options)) {
|
|
16
|
+
if (!key.startsWith('__')) {
|
|
17
|
+
result[key] = value;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return result;
|
|
21
|
+
}
|
|
22
|
+
function findRuntimeClass(mod) {
|
|
23
|
+
const isConnectorRuntimeClass = (val) => typeof val === 'function' &&
|
|
24
|
+
!!val.prototype?.sync &&
|
|
25
|
+
!!val.prototype?.execute;
|
|
26
|
+
const connector = Object.values(mod).find(isConnectorRuntimeClass);
|
|
27
|
+
if (connector)
|
|
28
|
+
return connector;
|
|
29
|
+
if (isConnectorRuntimeClass(mod.default))
|
|
30
|
+
return mod.default;
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// Auth-mode reverse channel: the parent process owns HTTP calls; the child
|
|
35
|
+
// talks to it via IPC. Each awaitSignal() call gets a unique request id.
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
let nextSignalRequestId = 1;
|
|
38
|
+
const pendingSignalWaiters = new Map();
|
|
39
|
+
const authAbortController = new AbortController();
|
|
40
|
+
function awaitAuthSignal(name, options) {
|
|
41
|
+
return new Promise((resolve, reject) => {
|
|
42
|
+
const requestId = nextSignalRequestId++;
|
|
43
|
+
pendingSignalWaiters.set(requestId, { resolve, reject });
|
|
44
|
+
let timer;
|
|
45
|
+
if (options?.timeoutMs && options.timeoutMs > 0) {
|
|
46
|
+
timer = setTimeout(() => {
|
|
47
|
+
pendingSignalWaiters.delete(requestId);
|
|
48
|
+
reject(new Error(`awaitSignal('${name}') timed out after ${options.timeoutMs}ms`));
|
|
49
|
+
}, options.timeoutMs);
|
|
50
|
+
}
|
|
51
|
+
const wrap = pendingSignalWaiters.get(requestId);
|
|
52
|
+
if (wrap) {
|
|
53
|
+
pendingSignalWaiters.set(requestId, {
|
|
54
|
+
resolve: (v) => {
|
|
55
|
+
if (timer)
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
wrap.resolve(v);
|
|
58
|
+
},
|
|
59
|
+
reject: (e) => {
|
|
60
|
+
if (timer)
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
wrap.reject(e);
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
void sendIPC({
|
|
67
|
+
type: 'await_signal_request',
|
|
68
|
+
requestId,
|
|
69
|
+
name,
|
|
70
|
+
timeoutMs: options?.timeoutMs ?? null,
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
async function executeConnectorRuntime(instance, context) {
|
|
75
|
+
const isAction = typeof context.options?.__action_key === 'string';
|
|
76
|
+
const isAuth = context.options?.__auth_mode === true;
|
|
77
|
+
const credentials = context.sessionState?.oauth ?? null;
|
|
78
|
+
const publicConfig = stripInternalOptions(context.options ?? {});
|
|
79
|
+
const runtimeConfig = {
|
|
80
|
+
...(context.env ?? {}),
|
|
81
|
+
...publicConfig,
|
|
82
|
+
};
|
|
83
|
+
if (isAuth) {
|
|
84
|
+
const authResult = await instance.authenticate({
|
|
85
|
+
config: (context.options?.__auth_config ?? {}),
|
|
86
|
+
previousCredentials: (context.options?.__auth_previous_credentials ?? null),
|
|
87
|
+
emit: async (artifact) => {
|
|
88
|
+
await sendIPC({ type: 'auth_artifact', artifact });
|
|
89
|
+
},
|
|
90
|
+
awaitSignal: awaitAuthSignal,
|
|
91
|
+
signal: authAbortController.signal,
|
|
92
|
+
});
|
|
93
|
+
if (!authResult?.credentials) {
|
|
94
|
+
throw new Error('authenticate() returned no credentials');
|
|
95
|
+
}
|
|
96
|
+
const result = {
|
|
97
|
+
contents: [],
|
|
98
|
+
checkpoint: null,
|
|
99
|
+
auth_result: {
|
|
100
|
+
credentials: authResult.credentials,
|
|
101
|
+
metadata: authResult.metadata,
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
return result;
|
|
105
|
+
}
|
|
106
|
+
if (isAction) {
|
|
107
|
+
const actionKey = context.options.__action_key;
|
|
108
|
+
const actionInput = (context.options.__action_input ?? {});
|
|
109
|
+
const actionResult = await instance.execute({
|
|
110
|
+
actionKey,
|
|
111
|
+
input: actionInput,
|
|
112
|
+
credentials,
|
|
113
|
+
config: runtimeConfig,
|
|
114
|
+
});
|
|
115
|
+
if (!actionResult?.success) {
|
|
116
|
+
throw new Error(actionResult?.error || `Action '${actionKey}' failed`);
|
|
117
|
+
}
|
|
118
|
+
const result = {
|
|
119
|
+
contents: [
|
|
120
|
+
{
|
|
121
|
+
origin_id: `action-${actionKey}-${Date.now()}`,
|
|
122
|
+
payload_text: '',
|
|
123
|
+
source_url: '',
|
|
124
|
+
occurred_at: new Date(),
|
|
125
|
+
score: 0,
|
|
126
|
+
metadata: actionResult.output ?? {},
|
|
127
|
+
},
|
|
128
|
+
],
|
|
129
|
+
checkpoint: context.checkpoint ?? null,
|
|
130
|
+
metadata: {
|
|
131
|
+
items_found: 0,
|
|
132
|
+
items_skipped: 0,
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
return result;
|
|
136
|
+
}
|
|
137
|
+
const emitEvents = async (events) => {
|
|
138
|
+
const normalized = normalizeEvents(events);
|
|
139
|
+
for (let index = 0; index < normalized.length; index += CONTENT_CHUNK_SIZE) {
|
|
140
|
+
await sendIPC({
|
|
141
|
+
type: 'content_chunk',
|
|
142
|
+
items: normalized.slice(index, index + CONTENT_CHUNK_SIZE),
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
const updateCheckpoint = async (checkpoint) => {
|
|
147
|
+
await sendIPC({ type: 'checkpoint_update', checkpoint: checkpoint ?? null });
|
|
148
|
+
};
|
|
149
|
+
const syncResult = (await instance.sync({
|
|
150
|
+
feedKey: context.options?.__feed_key,
|
|
151
|
+
config: runtimeConfig,
|
|
152
|
+
checkpoint: context.checkpoint ?? null,
|
|
153
|
+
credentials,
|
|
154
|
+
entityIds: context.options?.__entity_ids ?? [],
|
|
155
|
+
sessionState: context.sessionState ?? null,
|
|
156
|
+
emitEvents,
|
|
157
|
+
updateCheckpoint,
|
|
158
|
+
}));
|
|
159
|
+
const events = Array.isArray(syncResult?.events) ? syncResult.events : [];
|
|
160
|
+
await emitEvents(events);
|
|
161
|
+
const result = {
|
|
162
|
+
contents: [],
|
|
163
|
+
checkpoint: (syncResult?.checkpoint ?? null),
|
|
164
|
+
auth_update: syncResult?.auth_update ?? undefined,
|
|
165
|
+
metadata: {
|
|
166
|
+
items_found: typeof syncResult?.metadata?.items_found === 'number'
|
|
167
|
+
? syncResult.metadata.items_found
|
|
168
|
+
: events.length,
|
|
169
|
+
items_skipped: typeof syncResult?.metadata?.items_skipped === 'number'
|
|
170
|
+
? syncResult.metadata.items_skipped
|
|
171
|
+
: 0,
|
|
172
|
+
...(syncResult?.metadata ?? {}),
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
return result;
|
|
176
|
+
}
|
|
177
|
+
function normalizeEvents(events) {
|
|
178
|
+
return events.map((event) => normalizeEventEnvelope(event));
|
|
179
|
+
}
|
|
180
|
+
/** Send an IPC message and wait for it to be flushed to the parent. */
|
|
181
|
+
function sendIPC(msg) {
|
|
182
|
+
return new Promise((resolve, reject) => {
|
|
183
|
+
process.send(msg, (err) => {
|
|
184
|
+
if (err)
|
|
185
|
+
reject(err);
|
|
186
|
+
else
|
|
187
|
+
resolve();
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Best-effort handlers for top-level errors that previously escaped the
|
|
193
|
+
* runner's try/catch and surfaced as the bare wrapper string in the parent.
|
|
194
|
+
* These do NOT catch SIGKILL, native crashes, OOM, or sync `process.exit()`;
|
|
195
|
+
* those are surfaced via `exit_reason` in the parent SubprocessExecutor.
|
|
196
|
+
*/
|
|
197
|
+
function installUncaughtHandlers() {
|
|
198
|
+
let handled = false;
|
|
199
|
+
const safeStringify = (v) => {
|
|
200
|
+
try {
|
|
201
|
+
return JSON.stringify(v) ?? String(v);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return String(v);
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
const handle = async (err) => {
|
|
208
|
+
if (handled)
|
|
209
|
+
return;
|
|
210
|
+
handled = true;
|
|
211
|
+
const e = err instanceof Error ? err : new Error(typeof err === 'string' ? err : safeStringify(err));
|
|
212
|
+
try {
|
|
213
|
+
await sendIPC({
|
|
214
|
+
type: 'error',
|
|
215
|
+
error: {
|
|
216
|
+
message: e.message,
|
|
217
|
+
stack: e.stack,
|
|
218
|
+
name: e.name,
|
|
219
|
+
},
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
// If IPC is dead, the parent's exit-handler path will still produce
|
|
224
|
+
// diagnostics from the output tail.
|
|
225
|
+
}
|
|
226
|
+
finally {
|
|
227
|
+
process.exit(1);
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
process.on('uncaughtException', (err) => {
|
|
231
|
+
void handle(err);
|
|
232
|
+
});
|
|
233
|
+
process.on('unhandledRejection', (reason) => {
|
|
234
|
+
void handle(reason);
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
async function main() {
|
|
238
|
+
installUncaughtHandlers();
|
|
239
|
+
let started = false;
|
|
240
|
+
// Wait for message from parent
|
|
241
|
+
process.on('message', async (msg) => {
|
|
242
|
+
// Auth-mode reverse channel: parent sends signal payloads + abort.
|
|
243
|
+
if (msg?.type === 'await_signal_response') {
|
|
244
|
+
const waiter = pendingSignalWaiters.get(msg.requestId);
|
|
245
|
+
if (waiter) {
|
|
246
|
+
pendingSignalWaiters.delete(msg.requestId);
|
|
247
|
+
if (msg.error) {
|
|
248
|
+
waiter.reject(new Error(String(msg.error)));
|
|
249
|
+
}
|
|
250
|
+
else {
|
|
251
|
+
waiter.resolve((msg.signal ?? {}));
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
if (msg?.type === 'abort_auth') {
|
|
257
|
+
authAbortController.abort();
|
|
258
|
+
for (const [id, waiter] of pendingSignalWaiters.entries()) {
|
|
259
|
+
waiter.reject(new Error('Auth run aborted'));
|
|
260
|
+
pendingSignalWaiters.delete(id);
|
|
261
|
+
}
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (started)
|
|
265
|
+
return;
|
|
266
|
+
started = true;
|
|
267
|
+
// Keep temp module under cwd so bare imports (e.g. owletto) resolve via local node_modules.
|
|
268
|
+
const tmpFile = join(process.cwd(), `.connector-child-${process.pid}-${Date.now()}.mjs`);
|
|
269
|
+
try {
|
|
270
|
+
const { compiledCode, context } = msg;
|
|
271
|
+
// Write compiled code to temp file for dynamic import
|
|
272
|
+
await writeFile(tmpFile, compiledCode, 'utf-8');
|
|
273
|
+
// Import the compiled module
|
|
274
|
+
const mod = await import(pathToFileURL(tmpFile).href);
|
|
275
|
+
const RuntimeClass = findRuntimeClass(mod);
|
|
276
|
+
if (!RuntimeClass) {
|
|
277
|
+
throw new Error('No ConnectorRuntime class found. Expected a class with sync() and execute() methods.');
|
|
278
|
+
}
|
|
279
|
+
const instance = new RuntimeClass();
|
|
280
|
+
const result = await executeConnectorRuntime(instance, context);
|
|
281
|
+
// Send result back to parent (wait for IPC flush before exiting)
|
|
282
|
+
await sendIPC({ type: 'result', result });
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
if (error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
286
|
+
const pkgMatch = error.message.match(/Cannot find package '([^']+)'/);
|
|
287
|
+
const pkg = pkgMatch?.[1] ?? 'unknown';
|
|
288
|
+
error.message =
|
|
289
|
+
`Connector requires '${pkg}' but it's not installed in the runtime image. ` +
|
|
290
|
+
`'${pkg}' is declared as an external dependency in EXTERNAL_RUNTIME_DEPS ` +
|
|
291
|
+
`(packages/connector-worker/src/runtime-deps.ts). ` +
|
|
292
|
+
`Add it to packages/connector-worker/package.json and rebuild the runtime image.`;
|
|
293
|
+
}
|
|
294
|
+
await sendIPC({
|
|
295
|
+
type: 'error',
|
|
296
|
+
error: {
|
|
297
|
+
message: error.message,
|
|
298
|
+
stack: error.stack,
|
|
299
|
+
name: error.name,
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
finally {
|
|
304
|
+
// Clean up temp file
|
|
305
|
+
try {
|
|
306
|
+
await rm(tmpFile, { force: true });
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// Ignore cleanup errors
|
|
310
|
+
}
|
|
311
|
+
// Exit after sending result
|
|
312
|
+
process.exit(0);
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
main();
|
|
317
|
+
//# sourceMappingURL=child-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"child-runner.js","sourceRoot":"","sources":["../../src/executor/child-runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAEtD,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAa/B,SAAS,oBAAoB,CAAC,OAA4B;IACxD,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,GAA4B;IACpD,MAAM,uBAAuB,GAAG,CAAC,GAAY,EAAwB,EAAE,CACrE,OAAO,GAAG,KAAK,UAAU;QACzB,CAAC,CAAE,GAAW,CAAC,SAAS,EAAE,IAAI;QAC9B,CAAC,CAAE,GAAW,CAAC,SAAS,EAAE,OAAO,CAAC;IAEpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACnE,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,uBAAuB,CAAE,GAAW,CAAC,OAAO,CAAC;QAAE,OAAQ,GAAW,CAAC,OAAO,CAAC;IAE/E,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,yEAAyE;AACzE,8EAA8E;AAE9E,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAC5B,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAGjC,CAAC;AACJ,MAAM,mBAAmB,GAAG,IAAI,eAAe,EAAE,CAAC;AAElD,SAAS,eAAe,CACtB,IAAY,EACZ,OAAgC;IAEhC,OAAO,IAAI,OAAO,CAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC9D,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;QACxC,oBAAoB,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,KAAiC,CAAC;QACtC,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAChD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,oBAAoB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACvC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,IAAI,sBAAsB,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;YACrF,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,IAAI,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,IAAI,EAAE,CAAC;YACT,oBAAoB,CAAC,GAAG,CAAC,SAAS,EAAE;gBAClC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;oBACb,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;oBACZ,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC/B,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC;YACX,IAAI,EAAE,sBAAsB;YAC5B,SAAS;YACT,IAAI;YACJ,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI;SACtC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,QAAa,EAAE,OAAgC;IACpF,MAAM,QAAQ,GAAG,OAAO,OAAO,CAAC,OAAO,EAAE,YAAY,KAAK,QAAQ,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;IACrD,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,KAAK,IAAI,IAAI,CAAC;IACxD,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG;QACpB,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACtB,GAAG,YAAY;KAChB,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC;YAC7C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,IAAI,EAAE,CAA4B;YACzE,mBAAmB,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,2BAA2B,IAAI,IAAI,CAGlE;YACR,IAAI,EAAE,KAAK,EAAE,QAAiC,EAAE,EAAE;gBAChD,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC;YACrD,CAAC;YACD,WAAW,EAAE,eAAe;YAC5B,MAAM,EAAE,mBAAmB,CAAC,MAAM;SACnC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAmB;YAC7B,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE;gBACX,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;aAC9B;SACF,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;QAC/C,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAA4B,CAAC;QACtF,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC;YAC1C,SAAS;YACT,KAAK,EAAE,WAAW;YAClB,WAAW;YACX,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,YAAY,EAAE,KAAK,IAAI,WAAW,SAAS,UAAU,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,MAAM,GAAmB;YAC7B,QAAQ,EAAE;gBACR;oBACE,SAAS,EAAE,UAAU,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;oBAC9C,YAAY,EAAE,EAAE;oBAChB,UAAU,EAAE,EAAE;oBACd,WAAW,EAAE,IAAI,IAAI,EAAE;oBACvB,KAAK,EAAE,CAAC;oBACR,QAAQ,EAAE,YAAY,CAAC,MAAM,IAAI,EAAE;iBACpC;aACF;YACD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,QAAQ,EAAE;gBACR,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;aACjB;SACF,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,EAAE,MAAiB,EAAE,EAAE;QAC7C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAC3C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,kBAAkB,EAAE,CAAC;YAC3E,MAAM,OAAO,CAAC;gBACZ,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,kBAAkB,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,EAAE,UAA0C,EAAE,EAAE;QAC5E,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU;QACpC,MAAM,EAAE,aAAa;QACrB,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;QACtC,WAAW;QACX,SAAS,EAAG,OAAO,CAAC,OAAO,EAAE,YAAqC,IAAI,EAAE;QACxE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;QAC1C,UAAU;QACV,gBAAgB;KACjB,CAAC,CAAe,CAAC;IAElB,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,MAAM,GAAmB;QAC7B,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,CAAC,UAAU,EAAE,UAAU,IAAI,IAAI,CAAQ;QACnD,WAAW,EAAE,UAAU,EAAE,WAAW,IAAI,SAAS;QACjD,QAAQ,EAAE;YACR,WAAW,EACT,OAAO,UAAU,EAAE,QAAQ,EAAE,WAAW,KAAK,QAAQ;gBACnD,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,WAAW;gBACjC,CAAC,CAAC,MAAM,CAAC,MAAM;YACnB,aAAa,EACX,OAAO,UAAU,EAAE,QAAQ,EAAE,aAAa,KAAK,QAAQ;gBACrD,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,aAAa;gBACnC,CAAC,CAAC,CAAC;YACP,GAAG,CAAC,UAAU,EAAE,QAAQ,IAAI,EAAE,CAAC;SAChC;KACF,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,MAAiB;IACxC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,uEAAuE;AACvE,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,OAAO,CAAC,IAAK,CAAC,GAAG,EAAE,CAAC,GAAiB,EAAE,EAAE;YACvC,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;gBAChB,OAAO,EAAE,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB;IAC9B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,aAAa,GAAG,CAAC,CAAU,EAAU,EAAE;QAC3C,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,EAAE,GAAY,EAAE,EAAE;QACpC,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,GACL,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC;YACH,MAAM,OAAO,CAAC;gBACZ,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE;oBACL,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb;aACF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,oEAAoE;YACpE,oCAAoC;QACtC,CAAC;gBAAS,CAAC;YACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;QACtC,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,EAAE;QAC1C,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,uBAAuB,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;QACvC,mEAAmE;QACnE,IAAI,GAAG,EAAE,IAAI,KAAK,uBAAuB,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACvD,IAAI,MAAM,EAAE,CAAC;gBACX,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACd,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9C,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YAC/B,mBAAmB,CAAC,KAAK,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;gBAC7C,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QAEf,4FAA4F;QAC5F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEzF,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,GAAmB,CAAC;YAEtD,sDAAsD;YACtD,MAAM,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YAEhD,6BAA6B;YAC7B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC;YAEtD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,IAAK,YAA8B,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEhE,iEAAiE;YACjE,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBACtE,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;gBACvC,KAAK,CAAC,OAAO;oBACX,uBAAuB,GAAG,iDAAiD;wBAC3E,IAAI,GAAG,mEAAmE;wBAC1E,mDAAmD;wBACnD,iFAAiF,CAAC;YACtF,CAAC;YACD,MAAM,OAAO,CAAC;gBACZ,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE;oBACL,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB;aACF,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,qBAAqB;YACrB,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,4BAA4B;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { Checkpoint, Content, FeedOptions, SessionState } from '@lobu/connector-sdk';
|
|
2
|
+
/**
|
|
3
|
+
* Result shape returned by the subprocess executor.
|
|
4
|
+
* Mirrors the legacy SyncResult from the SDK (contents + checkpoint).
|
|
5
|
+
*/
|
|
6
|
+
export interface FeedSyncResult {
|
|
7
|
+
contents: Content[];
|
|
8
|
+
checkpoint: Checkpoint | null;
|
|
9
|
+
metadata?: Record<string, any>;
|
|
10
|
+
auth_update?: Record<string, any>;
|
|
11
|
+
/** Set when the subprocess completed an authenticate() run. */
|
|
12
|
+
auth_result?: {
|
|
13
|
+
credentials: Record<string, unknown>;
|
|
14
|
+
metadata?: Record<string, unknown>;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Context passed to the executor for a single sync job
|
|
19
|
+
*/
|
|
20
|
+
export interface SyncContext {
|
|
21
|
+
options: FeedOptions;
|
|
22
|
+
checkpoint: Checkpoint | null;
|
|
23
|
+
env: Record<string, string | undefined>;
|
|
24
|
+
sessionState?: SessionState | null;
|
|
25
|
+
apiType: 'api' | 'browser';
|
|
26
|
+
}
|
|
27
|
+
export interface ExecutionHooks {
|
|
28
|
+
onContentChunk?: (items: Content[]) => Promise<void> | void;
|
|
29
|
+
onCheckpointUpdate?: (checkpoint: Checkpoint | null) => Promise<void> | void;
|
|
30
|
+
collectContents?: boolean;
|
|
31
|
+
/** Auth runs: connector emits an artifact (QR/redirect/prompt/status). */
|
|
32
|
+
onAuthArtifact?: (artifact: Record<string, unknown>) => Promise<void> | void;
|
|
33
|
+
/** Auth runs: connector pauses until a named signal arrives. */
|
|
34
|
+
onAwaitAuthSignal?: (name: string, options?: {
|
|
35
|
+
timeoutMs?: number;
|
|
36
|
+
}) => Promise<Record<string, unknown>>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Pluggable executor interface
|
|
40
|
+
* Allows swapping between subprocess execution, direct execution, etc.
|
|
41
|
+
*/
|
|
42
|
+
export interface SyncExecutor {
|
|
43
|
+
execute(compiledCode: string, context: SyncContext, hooks?: ExecutionHooks): Promise<FeedSyncResult>;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=interface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/executor/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAE1F;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,UAAU,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAClC,+DAA+D;IAC/D,WAAW,CAAC,EAAE;QACZ,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,OAAO,EAAE,KAAK,GAAG,SAAS,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5D,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,0EAA0E;IAC1E,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7E,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,CAClB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,KAC7B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,CACL,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,WAAW,EACpB,KAAK,CAAC,EAAE,cAAc,GACrB,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interface.js","sourceRoot":"","sources":["../../src/executor/interface.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redact secrets from connector subprocess output before it leaves the worker.
|
|
3
|
+
*
|
|
4
|
+
* Patterns are deliberately broad — false positives are preferred to leaking a
|
|
5
|
+
* real credential into the runs table. Add new patterns here when a connector
|
|
6
|
+
* surfaces a new sensitive shape.
|
|
7
|
+
*/
|
|
8
|
+
export declare function redactOutput(text: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Streaming redactor for live tee to parent stdout/stderr. Buffers up to the
|
|
11
|
+
* last newline so that secrets split across stream chunk boundaries — for
|
|
12
|
+
* example "Authorization: Bear" + "er abc..." in two `data` events — still
|
|
13
|
+
* get matched by `redactOutput()`. The persisted `output_tail` already runs
|
|
14
|
+
* `redactOutput()` over the full ring-buffer string and is unaffected; this
|
|
15
|
+
* class exists solely to make the live-forwarded stream as safe as the
|
|
16
|
+
* persisted tail.
|
|
17
|
+
*
|
|
18
|
+
* `flush()` MUST be called on stream end to release any trailing partial
|
|
19
|
+
* line; otherwise its (redacted) content is dropped from the live tee but
|
|
20
|
+
* still appears in the persisted tail.
|
|
21
|
+
*/
|
|
22
|
+
export declare class StreamRedactor {
|
|
23
|
+
private carryover;
|
|
24
|
+
private static readonly MAX_BUFFER;
|
|
25
|
+
process(chunk: string, emit: (redacted: string) => void): void;
|
|
26
|
+
flush(emit: (redacted: string) => void): void;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=redact.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.d.ts","sourceRoot":"","sources":["../../src/executor/redact.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AA6DH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOjD;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,SAAS,CAAM;IAIvB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAE1C,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAuB9D,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;CAM9C"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redact secrets from connector subprocess output before it leaves the worker.
|
|
3
|
+
*
|
|
4
|
+
* Patterns are deliberately broad — false positives are preferred to leaking a
|
|
5
|
+
* real credential into the runs table. Add new patterns here when a connector
|
|
6
|
+
* surfaces a new sensitive shape.
|
|
7
|
+
*/
|
|
8
|
+
const REDACTED = '[REDACTED]';
|
|
9
|
+
// Value charset for assignment-style secrets: word + URL-safe base64 + the
|
|
10
|
+
// dot/dollar/percent that show up in OAuth tokens (e.g. ya29.a0AfH6SM…) and
|
|
11
|
+
// signed cookies. Excludes whitespace, quote, brace, and bracket so the
|
|
12
|
+
// pattern stops at the boundary of the value.
|
|
13
|
+
const SECRET_VALUE = `[\\w\\-.~+/=:%$]{12,}`;
|
|
14
|
+
const PATTERNS = [
|
|
15
|
+
// HTTP Authorization header (e.g. "Authorization: Bearer abc...") — match
|
|
16
|
+
// the rest of the line so multi-token schemes like "Bearer xxx" get caught.
|
|
17
|
+
{ regex: /Authorization:[^\r\n]+/gi, replacement: `Authorization: ${REDACTED}` },
|
|
18
|
+
// Cookie / Set-Cookie header — same line-eating shape.
|
|
19
|
+
{ regex: /(Set-)?Cookie:[^\r\n]+/gi, replacement: `$1Cookie: ${REDACTED}` },
|
|
20
|
+
// Bearer tokens anywhere (URL-safe-base64 style values)
|
|
21
|
+
{ regex: /Bearer\s+[\w\-.~+/=]+/gi, replacement: `Bearer ${REDACTED}` },
|
|
22
|
+
// JWT shape (eyJ...header.payload.sig)
|
|
23
|
+
{ regex: /eyJ[\w\-]+\.[\w\-]+\.[\w\-]+/g, replacement: REDACTED },
|
|
24
|
+
// Google OAuth access token shape (ya29.<varies>) — high-confidence.
|
|
25
|
+
{ regex: /ya29\.[\w\-.]{20,}/g, replacement: REDACTED },
|
|
26
|
+
// URI userinfo: `scheme://user:pass@host` — redact the password segment.
|
|
27
|
+
// Captures any scheme; replaces password while preserving structure.
|
|
28
|
+
{
|
|
29
|
+
regex: /([a-z][a-z0-9+\-.]*:\/\/[^:/\s]+):([^@/\s]+)@/gi,
|
|
30
|
+
replacement: `$1:${REDACTED}@`,
|
|
31
|
+
},
|
|
32
|
+
// CH_API_KEY=value (literal env-var key from the connector ecosystem)
|
|
33
|
+
{
|
|
34
|
+
regex: /(CH_API_KEY)(["'\s:=]+["']?)([\w\-]+)(["']?)/gi,
|
|
35
|
+
replacement: `$1$2${REDACTED}$4`,
|
|
36
|
+
},
|
|
37
|
+
// AWS_<SOMETHING>_KEY / AWS_<SOMETHING>_TOKEN env-style.
|
|
38
|
+
{
|
|
39
|
+
regex: new RegExp(`(AWS_[A-Z0-9_]*(?:KEY|TOKEN|SECRET))(["'\\s:=]+["']?)(${SECRET_VALUE})(["']?)`, 'g'),
|
|
40
|
+
replacement: `$1$2${REDACTED}$4`,
|
|
41
|
+
},
|
|
42
|
+
// JSON/YAML/env-var style `api_key=...`, `apikey: "..."`, `access_token: ...`,
|
|
43
|
+
// `secret = "..."`, `refresh_token: ...`, `id_token=...`, `_authToken: ...`,
|
|
44
|
+
// `password: "..."`, `client_secret=...`.
|
|
45
|
+
{
|
|
46
|
+
regex: new RegExp(`((?:api[_-]?key|apikey|access[_-]?token|refresh[_-]?token|id[_-]?token|_?auth[_-]?token|client[_-]?secret|secret|password))(["'\\s:=]+["']?)(${SECRET_VALUE})(["']?)`, 'gi'),
|
|
47
|
+
replacement: `$1$2${REDACTED}$4`,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
export function redactOutput(text) {
|
|
51
|
+
if (!text)
|
|
52
|
+
return text;
|
|
53
|
+
let result = text;
|
|
54
|
+
for (const { regex, replacement } of PATTERNS) {
|
|
55
|
+
result = result.replace(regex, replacement);
|
|
56
|
+
}
|
|
57
|
+
return result;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Streaming redactor for live tee to parent stdout/stderr. Buffers up to the
|
|
61
|
+
* last newline so that secrets split across stream chunk boundaries — for
|
|
62
|
+
* example "Authorization: Bear" + "er abc..." in two `data` events — still
|
|
63
|
+
* get matched by `redactOutput()`. The persisted `output_tail` already runs
|
|
64
|
+
* `redactOutput()` over the full ring-buffer string and is unaffected; this
|
|
65
|
+
* class exists solely to make the live-forwarded stream as safe as the
|
|
66
|
+
* persisted tail.
|
|
67
|
+
*
|
|
68
|
+
* `flush()` MUST be called on stream end to release any trailing partial
|
|
69
|
+
* line; otherwise its (redacted) content is dropped from the live tee but
|
|
70
|
+
* still appears in the persisted tail.
|
|
71
|
+
*/
|
|
72
|
+
export class StreamRedactor {
|
|
73
|
+
carryover = '';
|
|
74
|
+
// Safety cap for input with no newlines at all; emit the prefix and keep
|
|
75
|
+
// a sliding window of the last MAX_BUFFER chars to catch boundary splits
|
|
76
|
+
// up to that length.
|
|
77
|
+
static MAX_BUFFER = 8192;
|
|
78
|
+
process(chunk, emit) {
|
|
79
|
+
if (!chunk)
|
|
80
|
+
return;
|
|
81
|
+
const combined = this.carryover + chunk;
|
|
82
|
+
const lastNewline = combined.lastIndexOf('\n');
|
|
83
|
+
if (lastNewline >= 0) {
|
|
84
|
+
const complete = combined.slice(0, lastNewline + 1);
|
|
85
|
+
this.carryover = combined.slice(lastNewline + 1);
|
|
86
|
+
emit(redactOutput(complete));
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (combined.length > StreamRedactor.MAX_BUFFER) {
|
|
90
|
+
// No newline but we have to bound memory. Redact the whole buffer
|
|
91
|
+
// before emitting — slicing would re-introduce a boundary mid-secret.
|
|
92
|
+
// Carryover resets; the next chunk starts fresh, accepting that a
|
|
93
|
+
// secret split across the cap boundary may be redacted twice (safe)
|
|
94
|
+
// but never split within a regex match.
|
|
95
|
+
emit(redactOutput(combined));
|
|
96
|
+
this.carryover = '';
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
this.carryover = combined;
|
|
100
|
+
}
|
|
101
|
+
flush(emit) {
|
|
102
|
+
if (this.carryover) {
|
|
103
|
+
emit(redactOutput(this.carryover));
|
|
104
|
+
this.carryover = '';
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=redact.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.js","sourceRoot":"","sources":["../../src/executor/redact.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,QAAQ,GAAG,YAAY,CAAC;AAE9B,2EAA2E;AAC3E,4EAA4E;AAC5E,wEAAwE;AACxE,8CAA8C;AAC9C,MAAM,YAAY,GAAG,uBAAuB,CAAC;AAE7C,MAAM,QAAQ,GAAkD;IAC9D,0EAA0E;IAC1E,4EAA4E;IAC5E,EAAE,KAAK,EAAE,0BAA0B,EAAE,WAAW,EAAE,kBAAkB,QAAQ,EAAE,EAAE;IAEhF,uDAAuD;IACvD,EAAE,KAAK,EAAE,0BAA0B,EAAE,WAAW,EAAE,aAAa,QAAQ,EAAE,EAAE;IAE3E,wDAAwD;IACxD,EAAE,KAAK,EAAE,yBAAyB,EAAE,WAAW,EAAE,UAAU,QAAQ,EAAE,EAAE;IAEvE,uCAAuC;IACvC,EAAE,KAAK,EAAE,+BAA+B,EAAE,WAAW,EAAE,QAAQ,EAAE;IAEjE,qEAAqE;IACrE,EAAE,KAAK,EAAE,qBAAqB,EAAE,WAAW,EAAE,QAAQ,EAAE;IAEvD,yEAAyE;IACzE,qEAAqE;IACrE;QACE,KAAK,EAAE,iDAAiD;QACxD,WAAW,EAAE,MAAM,QAAQ,GAAG;KAC/B;IAED,sEAAsE;IACtE;QACE,KAAK,EAAE,gDAAgD;QACvD,WAAW,EAAE,OAAO,QAAQ,IAAI;KACjC;IAED,yDAAyD;IACzD;QACE,KAAK,EAAE,IAAI,MAAM,CACf,yDAAyD,YAAY,UAAU,EAC/E,GAAG,CACJ;QACD,WAAW,EAAE,OAAO,QAAQ,IAAI;KACjC;IAED,+EAA+E;IAC/E,6EAA6E;IAC7E,0CAA0C;IAC1C;QACE,KAAK,EAAE,IAAI,MAAM,CACf,gJAAgJ,YAAY,UAAU,EACtK,IAAI,CACL;QACD,WAAW,EAAE,OAAO,QAAQ,IAAI;KACjC;CACF,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC9C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,cAAc;IACjB,SAAS,GAAG,EAAE,CAAC;IACvB,yEAAyE;IACzE,yEAAyE;IACzE,qBAAqB;IACb,MAAM,CAAU,UAAU,GAAG,IAAI,CAAC;IAE1C,OAAO,CAAC,KAAa,EAAE,IAAgC;QACrD,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;YACpD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;YACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC7B,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC;YAChD,kEAAkE;YAClE,sEAAsE;YACtE,kEAAkE;YAClE,oEAAoE;YACpE,wCAAwC;YACxC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,IAAgC;QACpC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACtB,CAAC;IACH,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { Checkpoint, Content, SessionState } from '@lobu/connector-sdk';
|
|
2
|
+
import type { ExecutionHooks, FeedSyncResult, SyncExecutor } from './interface.js';
|
|
3
|
+
interface ConnectorOAuthCredentials {
|
|
4
|
+
provider: string;
|
|
5
|
+
accessToken: string;
|
|
6
|
+
refreshToken?: string | null;
|
|
7
|
+
expiresAt?: string | null;
|
|
8
|
+
scope?: string | null;
|
|
9
|
+
}
|
|
10
|
+
interface BaseExecutionParams {
|
|
11
|
+
compiledCode: string;
|
|
12
|
+
env?: Record<string, string | undefined>;
|
|
13
|
+
connectionCredentials?: Record<string, string | undefined> | null;
|
|
14
|
+
sessionState?: SessionState | null;
|
|
15
|
+
credentials?: ConnectorOAuthCredentials | null;
|
|
16
|
+
apiType?: 'api' | 'browser';
|
|
17
|
+
executor?: SyncExecutor;
|
|
18
|
+
hooks?: ExecutionHooks;
|
|
19
|
+
}
|
|
20
|
+
interface SyncConnectorExecutionParams extends BaseExecutionParams {
|
|
21
|
+
mode: 'sync';
|
|
22
|
+
config?: Record<string, unknown> | null;
|
|
23
|
+
checkpoint?: Checkpoint | null;
|
|
24
|
+
feedKey?: string | null;
|
|
25
|
+
entityIds?: number[] | null;
|
|
26
|
+
}
|
|
27
|
+
interface ActionConnectorExecutionParams extends BaseExecutionParams {
|
|
28
|
+
mode: 'action';
|
|
29
|
+
actionKey: string;
|
|
30
|
+
actionInput?: Record<string, unknown> | null;
|
|
31
|
+
checkpoint?: Checkpoint | null;
|
|
32
|
+
}
|
|
33
|
+
interface AuthConnectorExecutionParams extends BaseExecutionParams {
|
|
34
|
+
mode: 'authenticate';
|
|
35
|
+
/** Connector-specific auth input (rare). */
|
|
36
|
+
config?: Record<string, unknown> | null;
|
|
37
|
+
/** Existing credentials for re-auth flows. */
|
|
38
|
+
previousCredentials?: Record<string, unknown> | null;
|
|
39
|
+
}
|
|
40
|
+
type ConnectorExecutionParams = SyncConnectorExecutionParams | ActionConnectorExecutionParams | AuthConnectorExecutionParams;
|
|
41
|
+
export declare function executeCompiledConnector(params: ConnectorExecutionParams): Promise<FeedSyncResult>;
|
|
42
|
+
export declare function normalizeEventEnvelope(event: Record<string, any>): Content;
|
|
43
|
+
export declare function getActionOutput(result: FeedSyncResult): Record<string, unknown>;
|
|
44
|
+
export {};
|
|
45
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/executor/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAe,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAe,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAGhG,UAAU,yBAAyB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,UAAU,mBAAmB;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,qBAAqB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;IAClE,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,WAAW,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,UAAU,4BAA6B,SAAQ,mBAAmB;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxC,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CAC7B;AAED,UAAU,8BAA+B,SAAQ,mBAAmB;IAClE,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7C,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;CAChC;AAED,UAAU,4BAA6B,SAAQ,mBAAmB;IAChE,IAAI,EAAE,cAAc,CAAC;IACrB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxC,8CAA8C;IAC9C,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACtD;AAED,KAAK,wBAAwB,GACzB,4BAA4B,GAC5B,8BAA8B,GAC9B,4BAA4B,CAAC;AAqEjC,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,wBAAwB,GAC/B,OAAO,CAAC,cAAc,CAAC,CAIzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAgB1E;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE/E"}
|