@lobu/connector-worker 6.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +39 -0
  2. package/dist/__tests__/redact.test.d.ts +2 -0
  3. package/dist/__tests__/redact.test.d.ts.map +1 -0
  4. package/dist/__tests__/redact.test.js +135 -0
  5. package/dist/__tests__/redact.test.js.map +1 -0
  6. package/dist/bin.d.ts +11 -0
  7. package/dist/bin.d.ts.map +1 -0
  8. package/dist/bin.js +129 -0
  9. package/dist/bin.js.map +1 -0
  10. package/dist/daemon/client.d.ts +256 -0
  11. package/dist/daemon/client.d.ts.map +1 -0
  12. package/dist/daemon/client.js +152 -0
  13. package/dist/daemon/client.js.map +1 -0
  14. package/dist/daemon/executor.d.ts +25 -0
  15. package/dist/daemon/executor.d.ts.map +1 -0
  16. package/dist/daemon/executor.js +492 -0
  17. package/dist/daemon/executor.js.map +1 -0
  18. package/dist/daemon/index.d.ts +12 -0
  19. package/dist/daemon/index.d.ts.map +1 -0
  20. package/dist/daemon/index.js +9 -0
  21. package/dist/daemon/index.js.map +1 -0
  22. package/dist/daemon/worker.d.ts +54 -0
  23. package/dist/daemon/worker.d.ts.map +1 -0
  24. package/dist/daemon/worker.js +144 -0
  25. package/dist/daemon/worker.js.map +1 -0
  26. package/dist/embeddings.d.ts +9 -0
  27. package/dist/embeddings.d.ts.map +1 -0
  28. package/dist/embeddings.js +126 -0
  29. package/dist/embeddings.js.map +1 -0
  30. package/dist/executor/child-runner.d.ts +9 -0
  31. package/dist/executor/child-runner.d.ts.map +1 -0
  32. package/dist/executor/child-runner.js +317 -0
  33. package/dist/executor/child-runner.js.map +1 -0
  34. package/dist/executor/interface.d.ts +45 -0
  35. package/dist/executor/interface.d.ts.map +1 -0
  36. package/dist/executor/interface.js +2 -0
  37. package/dist/executor/interface.js.map +1 -0
  38. package/dist/executor/redact.d.ts +28 -0
  39. package/dist/executor/redact.d.ts.map +1 -0
  40. package/dist/executor/redact.js +108 -0
  41. package/dist/executor/redact.js.map +1 -0
  42. package/dist/executor/runtime.d.ts +45 -0
  43. package/dist/executor/runtime.d.ts.map +1 -0
  44. package/dist/executor/runtime.js +82 -0
  45. package/dist/executor/runtime.js.map +1 -0
  46. package/dist/executor/subprocess.d.ts +46 -0
  47. package/dist/executor/subprocess.d.ts.map +1 -0
  48. package/dist/executor/subprocess.js +378 -0
  49. package/dist/executor/subprocess.js.map +1 -0
  50. package/dist/index.d.ts +13 -0
  51. package/dist/index.d.ts.map +1 -0
  52. package/dist/index.js +12 -0
  53. package/dist/index.js.map +1 -0
  54. package/dist/runtime-deps.d.ts +33 -0
  55. package/dist/runtime-deps.d.ts.map +1 -0
  56. package/dist/runtime-deps.js +48 -0
  57. package/dist/runtime-deps.js.map +1 -0
  58. package/dist/types.d.ts +2 -0
  59. package/dist/types.d.ts.map +1 -0
  60. package/dist/types.js +2 -0
  61. package/dist/types.js.map +1 -0
  62. package/package.json +70 -0
@@ -0,0 +1,82 @@
1
+ import { SubprocessExecutor } from './subprocess.js';
2
+ function mergeExecutionEnv(env, connectionCredentials) {
3
+ return {
4
+ ...(env ?? {}),
5
+ ...(connectionCredentials ?? {}),
6
+ };
7
+ }
8
+ function mergeExecutionSessionState(sessionState, credentials) {
9
+ if (!sessionState && !credentials)
10
+ return null;
11
+ return {
12
+ ...(sessionState ?? {}),
13
+ ...(credentials ? { oauth: credentials } : {}),
14
+ };
15
+ }
16
+ function buildConnectorExecutionContext(params) {
17
+ const env = mergeExecutionEnv(params.env, params.connectionCredentials);
18
+ const sessionState = mergeExecutionSessionState(params.sessionState, params.credentials);
19
+ if (params.mode === 'action') {
20
+ return {
21
+ options: {
22
+ __action_key: params.actionKey,
23
+ __action_input: params.actionInput ?? {},
24
+ ...(params.actionInput ?? {}),
25
+ },
26
+ checkpoint: (params.checkpoint ?? null),
27
+ env,
28
+ sessionState,
29
+ apiType: params.apiType ?? 'api',
30
+ };
31
+ }
32
+ if (params.mode === 'authenticate') {
33
+ return {
34
+ options: {
35
+ __auth_mode: true,
36
+ __auth_config: params.config ?? {},
37
+ __auth_previous_credentials: params.previousCredentials ?? null,
38
+ },
39
+ checkpoint: null,
40
+ env,
41
+ sessionState,
42
+ apiType: params.apiType ?? 'api',
43
+ };
44
+ }
45
+ return {
46
+ options: {
47
+ ...(params.config ?? {}),
48
+ __feed_key: params.feedKey,
49
+ __entity_ids: params.entityIds ?? [],
50
+ },
51
+ checkpoint: (params.checkpoint ?? null),
52
+ env,
53
+ sessionState,
54
+ apiType: params.apiType ?? 'api',
55
+ };
56
+ }
57
+ export async function executeCompiledConnector(params) {
58
+ const executor = params.executor ?? new SubprocessExecutor();
59
+ const context = buildConnectorExecutionContext(params);
60
+ return executor.execute(params.compiledCode, context, params.hooks);
61
+ }
62
+ export function normalizeEventEnvelope(event) {
63
+ const originType = event.origin_type;
64
+ const rawDate = event.occurred_at ?? event.published_at;
65
+ return {
66
+ origin_id: event.origin_id ?? event.external_id,
67
+ payload_text: event.payload_text ?? event.content ?? '',
68
+ title: event.title,
69
+ author_name: event.author_name ?? event.author,
70
+ source_url: event.source_url ?? event.url ?? '',
71
+ occurred_at: rawDate ? new Date(rawDate) : new Date(),
72
+ origin_type: originType,
73
+ semantic_type: event.semantic_type ?? originType,
74
+ score: typeof event.score === 'number' ? event.score : 0,
75
+ origin_parent_id: event.origin_parent_id ?? event.parent_external_id ?? null,
76
+ metadata: event.metadata ?? {},
77
+ };
78
+ }
79
+ export function getActionOutput(result) {
80
+ return (result.contents[0]?.metadata ?? {});
81
+ }
82
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/executor/runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAiDrD,SAAS,iBAAiB,CACxB,GAAwC,EACxC,qBAAiE;IAEjE,OAAO;QACL,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;QACd,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CACjC,YAAkC,EAClC,WAA8C;IAE9C,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAE/C,OAAO;QACL,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED,SAAS,8BAA8B,CAAC,MAAgC;IACtE,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC;IACxE,MAAM,YAAY,GAAG,0BAA0B,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO;YACL,OAAO,EAAE;gBACP,YAAY,EAAE,MAAM,CAAC,SAAS;gBAC9B,cAAc,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;gBACxC,GAAI,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAA6B;aAC5C;YAChB,UAAU,EAAE,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAsB;YAC5D,GAAG;YACH,YAAY;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;SACjC,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE;gBACP,WAAW,EAAE,IAAI;gBACjB,aAAa,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;gBAClC,2BAA2B,EAAE,MAAM,CAAC,mBAAmB,IAAI,IAAI;aACjD;YAChB,UAAU,EAAE,IAAI;YAChB,GAAG;YACH,YAAY;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;SACjC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE;YACP,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YACxB,UAAU,EAAE,MAAM,CAAC,OAAO;YAC1B,YAAY,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;SACtB;QAChB,UAAU,EAAE,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAsB;QAC5D,GAAG;QACH,YAAY;QACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;KACjC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAgC;IAEhC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,kBAAkB,EAAE,CAAC;IAC7D,MAAM,OAAO,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA0B;IAC/D,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,YAAY,CAAC;IACxD,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,WAAW;QAC/C,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE;QACvD,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM;QAC9C,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,IAAI,EAAE;QAC/C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE;QACrD,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,UAAU;QAChD,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACxD,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,kBAAkB,IAAI,IAAI;QAC5E,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAsB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,EAAE,CAA4B,CAAC;AACzE,CAAC"}
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Subprocess Executor
3
+ *
4
+ * Executes compiled connector code in a forked child process.
5
+ * Provides process isolation between the worker and connector code.
6
+ * This is not a hardened security sandbox.
7
+ */
8
+ import type { ExecutionHooks, FeedSyncResult, SyncContext, SyncExecutor } from './interface.js';
9
+ /**
10
+ * exit_reason values surfaced to the runs table:
11
+ * - ok: successful 'result' IPC.
12
+ * - error_message: child sent {type:'error',...} via IPC.
13
+ * - timeout: parent killed the child with SIGKILL after timeoutMs.
14
+ * - oom: code !== 0 and output tail mentions a JS heap OOM.
15
+ * - crash: any other non-zero exit / unexpected signal.
16
+ */
17
+ export type SubprocessExitReason = 'ok' | 'error_message' | 'timeout' | 'oom' | 'crash';
18
+ /** Diagnostic fields attached to errors thrown by the executor. */
19
+ export interface SubprocessDiagnostics {
20
+ exitCode: number | null;
21
+ exitSignal: string | null;
22
+ outputTail: string;
23
+ exitReason: SubprocessExitReason;
24
+ }
25
+ export declare class SubprocessError extends Error implements SubprocessDiagnostics {
26
+ exitCode: number | null;
27
+ exitSignal: string | null;
28
+ outputTail: string;
29
+ exitReason: SubprocessExitReason;
30
+ constructor(message: string, diagnostics: SubprocessDiagnostics, options?: {
31
+ cause?: unknown;
32
+ });
33
+ }
34
+ interface SubprocessExecutorOptions {
35
+ /** Maximum execution time in ms (default: 600000 = 10 minutes) */
36
+ timeoutMs: number;
37
+ /** Max old space size for the child process in MB (default: 512) */
38
+ maxOldSpaceSize: number;
39
+ }
40
+ export declare class SubprocessExecutor implements SyncExecutor {
41
+ private options;
42
+ constructor(options?: Partial<SubprocessExecutorOptions>);
43
+ execute(compiledCode: string, context: SyncContext, hooks?: ExecutionHooks): Promise<FeedSyncResult>;
44
+ }
45
+ export {};
46
+ //# sourceMappingURL=subprocess.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"subprocess.d.ts","sourceRoot":"","sources":["../../src/executor/subprocess.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAGhG;;;;;;;GAOG;AACH,MAAM,MAAM,oBAAoB,GAAG,IAAI,GAAG,eAAe,GAAG,SAAS,GAAG,KAAK,GAAG,OAAO,CAAC;AAExF,mEAAmE;AACnE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,oBAAoB,CAAC;CAClC;AAED,qBAAa,eAAgB,SAAQ,KAAM,YAAW,qBAAqB;IACzE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,oBAAoB,CAAC;gBAG/B,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,qBAAqB,EAClC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAShC;AAqDD,UAAU,yBAAyB;IACjC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;CACzB;AAOD,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,OAAO,CAA4B;gBAE/B,OAAO,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAC;IAIlD,OAAO,CACX,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,WAAW,EACpB,KAAK,CAAC,EAAE,cAAc,GACrB,OAAO,CAAC,cAAc,CAAC;CAuT3B"}
@@ -0,0 +1,378 @@
1
+ /**
2
+ * Subprocess Executor
3
+ *
4
+ * Executes compiled connector code in a forked child process.
5
+ * Provides process isolation between the worker and connector code.
6
+ * This is not a hardened security sandbox.
7
+ */
8
+ import { fork } from 'node:child_process';
9
+ import { existsSync } from 'node:fs';
10
+ import { dirname, join } from 'node:path';
11
+ import { fileURLToPath } from 'node:url';
12
+ import { StreamRedactor, redactOutput } from './redact.js';
13
+ export class SubprocessError extends Error {
14
+ exitCode;
15
+ exitSignal;
16
+ outputTail;
17
+ exitReason;
18
+ constructor(message, diagnostics, options) {
19
+ super(message, options);
20
+ this.name = 'SubprocessError';
21
+ this.exitCode = diagnostics.exitCode;
22
+ this.exitSignal = diagnostics.exitSignal;
23
+ this.outputTail = diagnostics.outputTail;
24
+ this.exitReason = diagnostics.exitReason;
25
+ }
26
+ }
27
+ /** Per-stream ring buffer that preserves the most recent bytes. */
28
+ class RingBuffer {
29
+ cap;
30
+ chunks = [];
31
+ size = 0;
32
+ constructor(cap) {
33
+ this.cap = cap;
34
+ }
35
+ append(chunk) {
36
+ if (!chunk)
37
+ return;
38
+ this.chunks.push(chunk);
39
+ this.size += chunk.length;
40
+ while (this.size > this.cap && this.chunks.length > 0) {
41
+ const front = this.chunks[0];
42
+ const overflow = this.size - this.cap;
43
+ if (front.length <= overflow) {
44
+ this.size -= front.length;
45
+ this.chunks.shift();
46
+ }
47
+ else {
48
+ this.chunks[0] = front.slice(overflow);
49
+ this.size -= overflow;
50
+ }
51
+ }
52
+ }
53
+ toString() {
54
+ return this.chunks.join('');
55
+ }
56
+ }
57
+ const STREAM_TAIL_CAP_BYTES = 16 * 1024;
58
+ const __filename = fileURLToPath(import.meta.url);
59
+ const __dirname = dirname(__filename);
60
+ /** Only pass system env vars that child processes need for module resolution and basic operation. */
61
+ const SYSTEM_ENV_KEYS = [
62
+ 'PATH',
63
+ 'HOME',
64
+ 'TMPDIR',
65
+ 'TZ',
66
+ 'NODE_ENV',
67
+ 'NODE_PATH',
68
+ 'PLAYWRIGHT_BROWSERS_PATH',
69
+ ];
70
+ function pickSystemEnv() {
71
+ const env = {};
72
+ for (const key of SYSTEM_ENV_KEYS) {
73
+ if (process.env[key])
74
+ env[key] = process.env[key];
75
+ }
76
+ return env;
77
+ }
78
+ const DEFAULT_OPTIONS = {
79
+ timeoutMs: 600000,
80
+ maxOldSpaceSize: 512,
81
+ };
82
+ export class SubprocessExecutor {
83
+ options;
84
+ constructor(options) {
85
+ this.options = { ...DEFAULT_OPTIONS, ...options };
86
+ }
87
+ async execute(compiledCode, context, hooks) {
88
+ return new Promise((resolve, reject) => {
89
+ let childRunnerPath = join(__dirname, 'child-runner.js');
90
+ const childRunnerTsPath = join(__dirname, 'child-runner.ts');
91
+ const execArgv = [`--max-old-space-size=${this.options.maxOldSpaceSize}`];
92
+ const isBun = typeof process.versions.bun === 'string';
93
+ if (!existsSync(childRunnerPath) && existsSync(childRunnerTsPath)) {
94
+ childRunnerPath = childRunnerTsPath;
95
+ // Bun runs .ts natively. Loading tsx as an ESM hook on Bun fails
96
+ // with "Cannot find module './cjs/index.cjs' from ''" because tsx's
97
+ // loader.mjs invokes module.register('./cjs/index.cjs') with a parent
98
+ // URL that Bun's resolver treats as empty. Skip --import tsx on Bun.
99
+ if (!isBun)
100
+ execArgv.unshift('--import', 'tsx');
101
+ }
102
+ // Node subprocess execution is process isolation, not a security sandbox.
103
+ // Keep permission flags disabled unless the connector runtime is made compatible.
104
+ try {
105
+ const nodeVersion = parseInt(process.versions.node.split('.')[0], 10);
106
+ if (nodeVersion >= 20) {
107
+ // Uncomment only when the connector runtime is compatible with Node permissions:
108
+ // execArgv.push('--experimental-permission');
109
+ // execArgv.push(`--allow-fs-read=/tmp/*,${__dirname}/*`);
110
+ // execArgv.push('--allow-fs-write=/tmp/*');
111
+ }
112
+ }
113
+ catch {
114
+ // Ignore - permissions are best-effort
115
+ }
116
+ const child = fork(childRunnerPath, [], {
117
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
118
+ execArgv,
119
+ env: { ...pickSystemEnv(), ...context.env },
120
+ });
121
+ let resolved = false;
122
+ let terminalMessageReceived = false;
123
+ let timedOut = false;
124
+ let latestCheckpoint = context.checkpoint;
125
+ let finalMetadata;
126
+ let finalAuthUpdate;
127
+ let finalAuthResult;
128
+ const collectedContents = [];
129
+ const collectContents = hooks?.collectContents !== false;
130
+ let processingChain = Promise.resolve();
131
+ // Per-stream ring buffers — preserve the *tail* (most recent bytes),
132
+ // which is where the failure cause lands. Cap each at 16 KiB so a
133
+ // chatty connector can't grow the worker's memory.
134
+ const stdoutTail = new RingBuffer(STREAM_TAIL_CAP_BYTES);
135
+ const stderrTail = new RingBuffer(STREAM_TAIL_CAP_BYTES);
136
+ // Set timeout - kill child if it takes too long. timeoutMs <= 0 disables
137
+ // the timer (used for interactive auth runs that wait on human input).
138
+ const timeout = this.options.timeoutMs > 0
139
+ ? setTimeout(() => {
140
+ if (!resolved) {
141
+ console.error(`[SubprocessExecutor] Killing child process after ${this.options.timeoutMs}ms timeout`);
142
+ timedOut = true;
143
+ child.kill('SIGKILL');
144
+ }
145
+ }, this.options.timeoutMs)
146
+ : null;
147
+ const cleanup = () => {
148
+ if (timeout)
149
+ clearTimeout(timeout);
150
+ child.removeListener('message', onMessage);
151
+ child.removeListener('error', onError);
152
+ child.removeListener('exit', onExit);
153
+ child.stdout?.removeListener('data', onStdout);
154
+ child.stderr?.removeListener('data', onStderr);
155
+ // Flush any trailing partial line from each stream so the live tee
156
+ // matches what the persisted tail saw.
157
+ stdoutRedactor.flush((clean) => process.stdout.write(`[subprocess] ${clean}`));
158
+ stderrRedactor.flush((clean) => process.stderr.write(`[subprocess] ${clean}`));
159
+ };
160
+ const settle = (fn) => {
161
+ if (!resolved) {
162
+ resolved = true;
163
+ cleanup();
164
+ fn();
165
+ }
166
+ };
167
+ const queueTask = (task) => {
168
+ processingChain = processingChain.then(async () => {
169
+ await task();
170
+ });
171
+ processingChain.catch((err) => {
172
+ settle(() => {
173
+ child.kill('SIGKILL');
174
+ reject(err instanceof Error ? err : new Error(String(err)));
175
+ });
176
+ });
177
+ };
178
+ const combinedTail = () => {
179
+ const out = stdoutTail.toString();
180
+ const err = stderrTail.toString();
181
+ if (!out && !err)
182
+ return '';
183
+ if (!out)
184
+ return `[stderr]\n${err}`;
185
+ if (!err)
186
+ return `[stdout]\n${out}`;
187
+ return `[stdout]\n${out}\n[stderr]\n${err}`;
188
+ };
189
+ const computeExitReason = (tail) => {
190
+ if (timedOut)
191
+ return 'timeout';
192
+ if (/javascript heap out of memory/i.test(tail))
193
+ return 'oom';
194
+ return 'crash';
195
+ };
196
+ // Handle messages from child
197
+ const onMessage = (msg) => {
198
+ if (msg.type === 'content_chunk') {
199
+ const items = Array.isArray(msg.items) ? msg.items : [];
200
+ queueTask(async () => {
201
+ if (collectContents) {
202
+ collectedContents.push(...items);
203
+ }
204
+ await hooks?.onContentChunk?.(items);
205
+ });
206
+ return;
207
+ }
208
+ if (msg.type === 'checkpoint_update') {
209
+ latestCheckpoint = msg.checkpoint ?? null;
210
+ queueTask(async () => {
211
+ await hooks?.onCheckpointUpdate?.(latestCheckpoint);
212
+ });
213
+ return;
214
+ }
215
+ if (msg.type === 'auth_artifact') {
216
+ queueTask(async () => {
217
+ await hooks?.onAuthArtifact?.(msg.artifact ?? {});
218
+ });
219
+ return;
220
+ }
221
+ if (msg.type === 'await_signal_request') {
222
+ const requestId = msg.requestId;
223
+ const name = msg.name;
224
+ const timeoutMs = msg.timeoutMs ?? null;
225
+ queueTask(async () => {
226
+ if (!hooks?.onAwaitAuthSignal) {
227
+ try {
228
+ child.send({
229
+ type: 'await_signal_response',
230
+ requestId,
231
+ error: 'awaitSignal is not supported in this context',
232
+ });
233
+ }
234
+ catch {
235
+ /* ignore */
236
+ }
237
+ return;
238
+ }
239
+ try {
240
+ const signal = await hooks.onAwaitAuthSignal(name, {
241
+ timeoutMs: timeoutMs ?? undefined,
242
+ });
243
+ child.send({ type: 'await_signal_response', requestId, signal });
244
+ }
245
+ catch (err) {
246
+ child.send({
247
+ type: 'await_signal_response',
248
+ requestId,
249
+ error: err instanceof Error ? err.message : String(err),
250
+ });
251
+ }
252
+ });
253
+ return;
254
+ }
255
+ if (msg.type === 'result') {
256
+ terminalMessageReceived = true;
257
+ const result = msg.result;
258
+ queueTask(async () => {
259
+ finalMetadata = result.metadata;
260
+ finalAuthUpdate = result.auth_update;
261
+ finalAuthResult = result.auth_result;
262
+ latestCheckpoint = result.checkpoint;
263
+ if (Array.isArray(result.contents) && result.contents.length > 0) {
264
+ if (collectContents) {
265
+ collectedContents.push(...result.contents);
266
+ }
267
+ await hooks?.onContentChunk?.(result.contents);
268
+ }
269
+ settle(() => resolve({
270
+ contents: collectedContents,
271
+ checkpoint: latestCheckpoint,
272
+ metadata: finalMetadata,
273
+ auth_update: finalAuthUpdate,
274
+ auth_result: finalAuthResult,
275
+ }));
276
+ });
277
+ return;
278
+ }
279
+ if (msg.type === 'error') {
280
+ terminalMessageReceived = true;
281
+ const tail = redactOutput(combinedTail());
282
+ const diagnostics = {
283
+ exitCode: null,
284
+ exitSignal: null,
285
+ outputTail: tail,
286
+ exitReason: 'error_message',
287
+ };
288
+ // Connector code is allowed to throw with the offending value
289
+ // embedded — `throw new Error('failed with api_key=sk_live_…')`.
290
+ // Redact the message and stack the same way the persisted tail
291
+ // is redacted so secrets don't leak through the error path
292
+ // (which is also written to gateway logs by upstream callers).
293
+ const rawMessage = msg.error?.message ?? 'Subprocess reported error';
294
+ const error = new SubprocessError(redactOutput(rawMessage), diagnostics);
295
+ error.name = msg.error?.name ?? 'SubprocessError';
296
+ if (msg.error?.stack)
297
+ error.stack = redactOutput(msg.error.stack);
298
+ settle(() => reject(error));
299
+ return;
300
+ }
301
+ };
302
+ // Handle child errors
303
+ const onError = (err) => {
304
+ const tail = redactOutput(combinedTail());
305
+ const diagnostics = {
306
+ exitCode: null,
307
+ exitSignal: null,
308
+ outputTail: tail,
309
+ exitReason: 'crash',
310
+ };
311
+ const wrapped = new SubprocessError(`Subprocess error: ${err.message}`, diagnostics, {
312
+ cause: err,
313
+ });
314
+ settle(() => reject(wrapped));
315
+ };
316
+ // Handle child exit (single handler for both timeout cleanup and unexpected exits)
317
+ const onExit = (code, signal) => {
318
+ if (terminalMessageReceived) {
319
+ return;
320
+ }
321
+ settle(() => {
322
+ const tail = redactOutput(combinedTail());
323
+ const reason = computeExitReason(tail);
324
+ const prefix = reason === 'timeout'
325
+ ? `Feed execution timed out after ${this.options.timeoutMs}ms`
326
+ : reason === 'oom'
327
+ ? `Subprocess out of memory (code ${code}, signal ${signal})`
328
+ : `Subprocess exited with code ${code}, signal ${signal}`;
329
+ const message = tail ? `${prefix}\n${tail}` : prefix;
330
+ const diagnostics = {
331
+ exitCode: code,
332
+ exitSignal: signal,
333
+ outputTail: tail,
334
+ exitReason: reason,
335
+ };
336
+ reject(new SubprocessError(message, diagnostics));
337
+ });
338
+ };
339
+ // Forward child stdout to parent stdout for live tailing AND tap into
340
+ // the ring buffer so we can surface the tail on failure. Without this
341
+ // listener, stdio: 'pipe' fills the OS pipe buffer (~16-64 KB) and the
342
+ // child blocks on its next console.log until SIGKILL.
343
+ // Stream redactors buffer up to the last newline so secrets split
344
+ // across chunk boundaries still match. Persisted tails already
345
+ // redact the full ring-buffer string and are unaffected.
346
+ const stdoutRedactor = new StreamRedactor();
347
+ const stderrRedactor = new StreamRedactor();
348
+ const onStdout = (data) => {
349
+ const text = data.toString();
350
+ stdoutTail.append(text);
351
+ stdoutRedactor.process(text, (clean) => process.stdout.write(`[subprocess] ${clean}`));
352
+ };
353
+ // Forward child stderr to parent stderr for logging + ring buffer.
354
+ const onStderr = (data) => {
355
+ const text = data.toString();
356
+ stderrTail.append(text);
357
+ stderrRedactor.process(text, (clean) => process.stderr.write(`[subprocess] ${clean}`));
358
+ };
359
+ child.on('message', onMessage);
360
+ child.on('error', onError);
361
+ child.on('exit', onExit);
362
+ child.stdout?.on('data', onStdout);
363
+ child.stderr?.on('data', onStderr);
364
+ // Send the compiled code and context to the child
365
+ child.send({
366
+ compiledCode,
367
+ context: {
368
+ options: context.options,
369
+ checkpoint: context.checkpoint,
370
+ env: context.env,
371
+ sessionState: context.sessionState,
372
+ apiType: context.apiType,
373
+ },
374
+ });
375
+ });
376
+ }
377
+ }
378
+ //# sourceMappingURL=subprocess.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"subprocess.js","sourceRoot":"","sources":["../../src/executor/subprocess.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAoB3D,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAgB;IACxB,UAAU,CAAgB;IAC1B,UAAU,CAAS;IACnB,UAAU,CAAuB;IAEjC,YACE,OAAe,EACf,WAAkC,EAClC,OAA6B;QAE7B,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;IAC3C,CAAC;CACF;AAED,mEAAmE;AACnE,MAAM,UAAU;IAGe;IAFrB,MAAM,GAAa,EAAE,CAAC;IACtB,IAAI,GAAG,CAAC,CAAC;IACjB,YAA6B,GAAW;QAAX,QAAG,GAAH,GAAG,CAAQ;IAAG,CAAC;IAE5C,MAAM,CAAC,KAAa;QAClB,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;YACtC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACvC,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;CACF;AAED,MAAM,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;AAExC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAEtC,qGAAqG;AACrG,MAAM,eAAe,GAAG;IACtB,MAAM;IACN,MAAM;IACN,QAAQ;IACR,IAAI;IACJ,UAAU;IACV,WAAW;IACX,0BAA0B;CAC3B,CAAC;AACF,SAAS,aAAa;IACpB,MAAM,GAAG,GAAuC,EAAE,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QAClC,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AASD,MAAM,eAAe,GAA8B;IACjD,SAAS,EAAE,MAAM;IACjB,eAAe,EAAE,GAAG;CACrB,CAAC;AAEF,MAAM,OAAO,kBAAkB;IACrB,OAAO,CAA4B;IAE3C,YAAY,OAA4C;QACtD,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,YAAoB,EACpB,OAAoB,EACpB,KAAsB;QAEtB,OAAO,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrD,IAAI,eAAe,GAAG,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;YACzD,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;YAE7D,MAAM,QAAQ,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YAC1E,MAAM,KAAK,GAAG,OAAQ,OAAO,CAAC,QAA6B,CAAC,GAAG,KAAK,QAAQ,CAAC;YAC7E,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAClE,eAAe,GAAG,iBAAiB,CAAC;gBACpC,iEAAiE;gBACjE,oEAAoE;gBACpE,sEAAsE;gBACtE,qEAAqE;gBACrE,IAAI,CAAC,KAAK;oBAAE,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,CAAC;YAED,0EAA0E;YAC1E,kFAAkF;YAClF,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtE,IAAI,WAAW,IAAI,EAAE,EAAE,CAAC;oBACtB,iFAAiF;oBACjF,8CAA8C;oBAC9C,0DAA0D;oBAC1D,4CAA4C;gBAC9C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE;gBACtC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;gBACtC,QAAQ;gBACR,GAAG,EAAE,EAAE,GAAG,aAAa,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAuB;aACjE,CAAC,CAAC;YAEH,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,uBAAuB,GAAG,KAAK,CAAC;YACpC,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC;YAC1C,IAAI,aAA8C,CAAC;YACnD,IAAI,eAAgD,CAAC;YACrD,IAAI,eAA0D,CAAC;YAC/D,MAAM,iBAAiB,GAA+B,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,KAAK,EAAE,eAAe,KAAK,KAAK,CAAC;YACzD,IAAI,eAAe,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAExC,qEAAqE;YACrE,kEAAkE;YAClE,mDAAmD;YACnD,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,qBAAqB,CAAC,CAAC;YACzD,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAEzD,yEAAyE;YACzE,uEAAuE;YACvE,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;gBACxB,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE;oBACd,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CACX,oDAAoD,IAAI,CAAC,OAAO,CAAC,SAAS,YAAY,CACvF,CAAC;wBACF,QAAQ,GAAG,IAAI,CAAC;wBAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC5B,CAAC,CAAC,IAAI,CAAC;YAEX,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,IAAI,OAAO;oBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;gBACnC,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBAC3C,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBACvC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC/C,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC/C,mEAAmE;gBACnE,uCAAuC;gBACvC,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC/E,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC,CAAC;YACjF,CAAC,CAAC;YAEF,MAAM,MAAM,GAAG,CAAC,EAAc,EAAE,EAAE;gBAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,EAAE,CAAC;oBACV,EAAE,EAAE,CAAC;gBACP,CAAC;YACH,CAAC,CAAC;YAEF,MAAM,SAAS,GAAG,CAAC,IAAgC,EAAE,EAAE;gBACrD,eAAe,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;oBAChD,MAAM,IAAI,EAAE,CAAC;gBACf,CAAC,CAAC,CAAC;gBACH,eAAe,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC5B,MAAM,CAAC,GAAG,EAAE;wBACV,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;wBACtB,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAC9D,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;YAEF,MAAM,YAAY,GAAG,GAAW,EAAE;gBAChC,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG;oBAAE,OAAO,EAAE,CAAC;gBAC5B,IAAI,CAAC,GAAG;oBAAE,OAAO,aAAa,GAAG,EAAE,CAAC;gBACpC,IAAI,CAAC,GAAG;oBAAE,OAAO,aAAa,GAAG,EAAE,CAAC;gBACpC,OAAO,aAAa,GAAG,eAAe,GAAG,EAAE,CAAC;YAC9C,CAAC,CAAC;YAEF,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAwB,EAAE;gBAC/D,IAAI,QAAQ;oBAAE,OAAO,SAAS,CAAC;gBAC/B,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC9D,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC;YAEF,6BAA6B;YAC7B,MAAM,SAAS,GAAG,CAAC,GAAQ,EAAE,EAAE;gBAC7B,IAAI,GAAG,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;oBACjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxD,SAAS,CAAC,KAAK,IAAI,EAAE;wBACnB,IAAI,eAAe,EAAE,CAAC;4BACpB,iBAAiB,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;wBACnC,CAAC;wBACD,MAAM,KAAK,EAAE,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC;oBACvC,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;oBACrC,gBAAgB,GAAG,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC;oBAC1C,SAAS,CAAC,KAAK,IAAI,EAAE;wBACnB,MAAM,KAAK,EAAE,kBAAkB,EAAE,CAAC,gBAAgB,CAAC,CAAC;oBACtD,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;oBACjC,SAAS,CAAC,KAAK,IAAI,EAAE;wBACnB,MAAM,KAAK,EAAE,cAAc,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;oBACpD,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;oBACxC,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;oBAChC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;oBACtB,MAAM,SAAS,GAAkB,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC;oBACvD,SAAS,CAAC,KAAK,IAAI,EAAE;wBACnB,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,CAAC;4BAC9B,IAAI,CAAC;gCACH,KAAK,CAAC,IAAI,CAAC;oCACT,IAAI,EAAE,uBAAuB;oCAC7B,SAAS;oCACT,KAAK,EAAE,8CAA8C;iCACtD,CAAC,CAAC;4BACL,CAAC;4BAAC,MAAM,CAAC;gCACP,YAAY;4BACd,CAAC;4BACD,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC;4BACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE;gCACjD,SAAS,EAAE,SAAS,IAAI,SAAS;6BAClC,CAAC,CAAC;4BACH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;wBACnE,CAAC;wBAAC,OAAO,GAAG,EAAE,CAAC;4BACb,KAAK,CAAC,IAAI,CAAC;gCACT,IAAI,EAAE,uBAAuB;gCAC7B,SAAS;gCACT,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;6BACxD,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC1B,uBAAuB,GAAG,IAAI,CAAC;oBAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,MAAwB,CAAC;oBAC5C,SAAS,CAAC,KAAK,IAAI,EAAE;wBACnB,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC;wBAChC,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC;wBACrC,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC;wBACrC,gBAAgB,GAAG,MAAM,CAAC,UAAU,CAAC;wBAErC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACjE,IAAI,eAAe,EAAE,CAAC;gCACpB,iBAAiB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;4BAC7C,CAAC;4BACD,MAAM,KAAK,EAAE,cAAc,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;wBACjD,CAAC;wBAED,MAAM,CAAC,GAAG,EAAE,CACV,OAAO,CAAC;4BACN,QAAQ,EAAE,iBAAiB;4BAC3B,UAAU,EAAE,gBAAgB;4BAC5B,QAAQ,EAAE,aAAa;4BACvB,WAAW,EAAE,eAAe;4BAC5B,WAAW,EAAE,eAAe;yBAC7B,CAAC,CACH,CAAC;oBACJ,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACzB,uBAAuB,GAAG,IAAI,CAAC;oBAC/B,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,EAAE,CAAC,CAAC;oBAC1C,MAAM,WAAW,GAA0B;wBACzC,QAAQ,EAAE,IAAI;wBACd,UAAU,EAAE,IAAI;wBAChB,UAAU,EAAE,IAAI;wBAChB,UAAU,EAAE,eAAe;qBAC5B,CAAC;oBACF,8DAA8D;oBAC9D,iEAAiE;oBACjE,+DAA+D;oBAC/D,2DAA2D;oBAC3D,+DAA+D;oBAC/D,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,EAAE,OAAO,IAAI,2BAA2B,CAAC;oBACrE,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAC;oBACzE,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,IAAI,iBAAiB,CAAC;oBAClD,IAAI,GAAG,CAAC,KAAK,EAAE,KAAK;wBAAE,KAAK,CAAC,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAClE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBAC5B,OAAO;gBACT,CAAC;YACH,CAAC,CAAC;YAEF,sBAAsB;YACtB,MAAM,OAAO,GAAG,CAAC,GAAU,EAAE,EAAE;gBAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,EAAE,CAAC,CAAC;gBAC1C,MAAM,WAAW,GAA0B;oBACzC,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,UAAU,EAAE,IAAI;oBAChB,UAAU,EAAE,OAAO;iBACpB,CAAC;gBACF,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,qBAAqB,GAAG,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE;oBACnF,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;gBACH,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAChC,CAAC,CAAC;YAEF,mFAAmF;YACnF,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,MAAqB,EAAE,EAAE;gBAC5D,IAAI,uBAAuB,EAAE,CAAC;oBAC5B,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,GAAG,EAAE;oBACV,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,EAAE,CAAC,CAAC;oBAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBACvC,MAAM,MAAM,GACV,MAAM,KAAK,SAAS;wBAClB,CAAC,CAAC,kCAAkC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI;wBAC9D,CAAC,CAAC,MAAM,KAAK,KAAK;4BAChB,CAAC,CAAC,kCAAkC,IAAI,YAAY,MAAM,GAAG;4BAC7D,CAAC,CAAC,+BAA+B,IAAI,YAAY,MAAM,EAAE,CAAC;oBAChE,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;oBACrD,MAAM,WAAW,GAA0B;wBACzC,QAAQ,EAAE,IAAI;wBACd,UAAU,EAAE,MAAM;wBAClB,UAAU,EAAE,IAAI;wBAChB,UAAU,EAAE,MAAM;qBACnB,CAAC;oBACF,MAAM,CAAC,IAAI,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;gBACpD,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;YAEF,sEAAsE;YACtE,sEAAsE;YACtE,uEAAuE;YACvE,sDAAsD;YACtD,kEAAkE;YAClE,+DAA+D;YAC/D,yDAAyD;YACzD,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;YAC5C,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;YAE5C,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAE;gBAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC7B,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACxB,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC,CAAC;YACzF,CAAC,CAAC;YAEF,mEAAmE;YACnE,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAE;gBAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC7B,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACxB,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC,CAAC;YACzF,CAAC,CAAC;YAEF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC/B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC3B,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACzB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACnC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEnC,kDAAkD;YAClD,KAAK,CAAC,IAAI,CAAC;gBACT,YAAY;gBACZ,OAAO,EAAE;oBACP,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,YAAY,EAAE,OAAO,CAAC,YAAY;oBAClC,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * @owletto/worker
3
+ *
4
+ * Self-hosted worker for content intelligence.
5
+ * Includes subprocess executor and embedding generation.
6
+ *
7
+ * Usage:
8
+ * connector-worker daemon --api-url https://api.example.com
9
+ */
10
+ export type { CompleteRequest, ContentItem, DaemonConfig, ExecutorConfig, PollResponse, StreamBatch, WorkerCapabilities, } from './daemon/index.js';
11
+ export { executeRun, startDaemon, WorkerClient, WorkerDaemon } from './daemon/index.js';
12
+ export type { Env } from './types.js';
13
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,YAAY,EACV,eAAe,EACf,WAAW,EACX,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGxF,YAAY,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,12 @@
1
+ /**
2
+ * @owletto/worker
3
+ *
4
+ * Self-hosted worker for content intelligence.
5
+ * Includes subprocess executor and embedding generation.
6
+ *
7
+ * Usage:
8
+ * connector-worker daemon --api-url https://api.example.com
9
+ */
10
+ // Worker Daemon
11
+ export { executeRun, startDaemon, WorkerClient, WorkerDaemon } from './daemon/index.js';
12
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAWH,gBAAgB;AAChB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Single source of truth for npm packages that connector code may import but
3
+ * which we deliberately do NOT bundle into the compiled connector artifact.
4
+ *
5
+ * These deps must be installed in every runtime that executes compiled
6
+ * connectors — the server container that hosts in-process feed sync,
7
+ * and the connector-worker daemon that runs out-of-process. They appear in:
8
+ *
9
+ * - `server/src/utils/connector-compiler.ts` `external` list
10
+ * (esbuild leaves the imports as bare specifiers in the bundle)
11
+ * - `packages/connector-worker/package.json` dependencies (so the runtime can
12
+ * resolve them)
13
+ * - `assertExternalDepsResolvable()` (boot-time check that crashes loud
14
+ * instead of failing silently per-feed)
15
+ *
16
+ * Rule of thumb: only externalize deps that genuinely can't be bundled —
17
+ * native binaries (`sharp`, `jimp`) or runtime install steps
18
+ * (`playwright` ships browsers via `npx playwright install`). Pure JS deps
19
+ * like `pino` or `link-preview-js` should be bundled instead, even if it
20
+ * costs a few hundred KB per connector — bundling eliminates the entire
21
+ * class of "compiled connector references X but X isn't installed in the
22
+ * worker image" outages.
23
+ */
24
+ export declare const EXTERNAL_RUNTIME_DEPS: readonly ["playwright", "sharp", "jimp"];
25
+ export type ExternalRuntimeDep = (typeof EXTERNAL_RUNTIME_DEPS)[number];
26
+ /**
27
+ * Verify that every external runtime dep is resolvable from the current
28
+ * process. Call this once at startup of any service that executes compiled
29
+ * connectors. Throws (so the process crashes) instead of letting individual
30
+ * feed runs fail with `Missing npm dependency: X`.
31
+ */
32
+ export declare function assertExternalDepsResolvable(resolve: (specifier: string) => void): void;
33
+ //# sourceMappingURL=runtime-deps.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime-deps.d.ts","sourceRoot":"","sources":["../src/runtime-deps.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,qBAAqB,0CAA2C,CAAC;AAE9E,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;AAExE;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,GACnC,IAAI,CAiBN"}