luxray-agent 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,7 @@
1
+ // ../streaming/dist/config.js
2
+ import { existsSync, readFileSync, readdirSync } from "fs";
3
+ import { join } from "path";
4
+
1
5
  // ../streaming/dist/errors.js
2
6
  var StreamUnavailableError = class extends Error {
3
7
  brokers;
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../streaming/src/errors.ts","../../../streaming/src/config.ts","../../../streaming/src/envelope.ts","../../../streaming/src/kafka.ts","../../../streaming/src/topics.ts","../../src/client/task-stream.ts"],"sourcesContent":["/** Raised when the broker cannot be reached or refuses the connection. */\nexport class StreamUnavailableError extends Error {\n public override readonly name = 'StreamUnavailableError';\n\n constructor(\n message: string,\n public readonly brokers: readonly string[],\n public override readonly cause?: unknown,\n ) {\n super(message);\n }\n}\n\n/** Raised when a record is not a valid envelope. Never inferred as meaning. */\nexport class StreamProtocolError extends Error {\n public override readonly name = 'StreamProtocolError';\n}\n","import { StreamUnavailableError } from './errors.js';\n\nexport interface StreamSaslConfig {\n readonly mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';\n readonly username: string;\n readonly password: string;\n}\n\nexport interface StreamConfig {\n readonly brokers: readonly string[];\n readonly clientId: string;\n readonly ssl: boolean;\n readonly sasl: StreamSaslConfig | null;\n /** How long a broker connection may take before it counts as unavailable. */\n readonly connectionTimeoutMs: number;\n /** Retries per connect attempt; kept small so a missing broker fails fast. */\n readonly retryCount: number;\n /** Partitions used when a stream topic has to be created. Default 3. */\n readonly topicPartitions?: number;\n /** Replication factor used when a stream topic has to be created. Default 1. */\n readonly replicationFactor?: number;\n}\n\nconst DEFAULT_BROKERS = '127.0.0.1:9092';\n\nexport function streamConfigFromEnv(\n env: NodeJS.ProcessEnv = process.env,\n overrides: Partial<StreamConfig> = {},\n): StreamConfig {\n const brokers = (env['KAFKA_BROKERS'] ?? DEFAULT_BROKERS)\n .split(',')\n .map((broker) => broker.trim())\n .filter((broker) => broker.length > 0);\n\n if (brokers.length === 0) {\n throw new Error('KAFKA_BROKERS is set but lists no broker address.');\n }\n\n const merged: StreamConfig = {\n brokers,\n clientId: env['KAFKA_CLIENT_ID'] ?? 'luxray',\n ssl: (env['KAFKA_SSL'] ?? '').toLowerCase() === 'true',\n sasl: saslFromEnv(env),\n connectionTimeoutMs: 5_000,\n retryCount: 2,\n topicPartitions: positiveInt(env['KAFKA_TOPIC_PARTITIONS'], 3, 'KAFKA_TOPIC_PARTITIONS'),\n replicationFactor: positiveInt(\n env['KAFKA_REPLICATION_FACTOR'],\n 1,\n 'KAFKA_REPLICATION_FACTOR',\n ),\n ...overrides,\n };\n\n // A retry count of zero is rejected by kafkajs whenever the producer is\n // idempotent, which it always is here.\n return { ...merged, retryCount: Math.max(1, merged.retryCount) };\n}\n\nfunction saslFromEnv(env: NodeJS.ProcessEnv): StreamSaslConfig | null {\n const username = env['KAFKA_SASL_USERNAME'];\n const password = env['KAFKA_SASL_PASSWORD'];\n if (!username || !password) return null;\n\n const raw = (env['KAFKA_SASL_MECHANISM'] ?? 'plain').toLowerCase();\n if (raw !== 'plain' && raw !== 'scram-sha-256' && raw !== 'scram-sha-512') {\n throw new Error(\n `KAFKA_SASL_MECHANISM must be plain, scram-sha-256 or scram-sha-512 (got ${raw}).`,\n );\n }\n return { mechanism: raw, username, password };\n}\n\n/** Parses a positive integer setting, naming the variable in the error. */\nfunction positiveInt(raw: string | undefined, fallback: number, name: string): number {\n if (raw === undefined || raw.trim() === '') return fallback;\n const value = Number(raw);\n if (!Number.isInteger(value) || value < 1) {\n throw new Error(`${name} must be a positive integer (got \"${raw}\").`);\n }\n return value;\n}\n\n/** One error message for both processes, naming the thing to fix. */\nexport function unavailableError(config: StreamConfig, cause: unknown): StreamUnavailableError {\n const detail = cause instanceof Error ? cause.message : String(cause);\n return new StreamUnavailableError(\n `Kafka is not reachable at ${config.brokers.join(', ')} (${detail}). ` +\n 'Start it with `docker compose up -d kafka`, or point KAFKA_BROKERS at a broker.',\n config.brokers,\n cause,\n );\n}\n","import type { StreamEnvelope, StreamKind } from '@luxray/shared';\nimport { StreamProtocolError } from './errors.js';\n\nconst KINDS: readonly StreamKind[] = ['task', 'report', 'event', 'result'];\n\nexport function encodeEnvelope<T>(envelope: StreamEnvelope<T>): string {\n return JSON.stringify(envelope);\n}\n\n/**\n * Parses one record.\n *\n * A malformed record is a protocol error, never a partially usable message:\n * acting on half an offer would be worse than dropping it, because the durable\n * offer stays in the task row and is replayed on the next connection.\n *\n * `expected` may name several kinds when a topic deliberately carries more than\n * one. Page results share the report topic so a task's pages and its completion\n * stay in one partition order; the caller dispatches on the returned kind.\n */\nexport function decodeEnvelope<T>(\n raw: string,\n expected?: StreamKind | readonly StreamKind[],\n): StreamEnvelope<T> {\n let parsed: unknown;\n try {\n parsed = JSON.parse(raw);\n } catch (error) {\n throw new StreamProtocolError(`Record is not JSON: ${(error as Error).message}`);\n }\n\n if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {\n throw new StreamProtocolError('Record is not an object.');\n }\n\n const record = parsed as Record<string, unknown>;\n if (record['envelope'] !== 1) {\n throw new StreamProtocolError(`Unsupported envelope version: ${String(record['envelope'])}.`);\n }\n\n const kind = record['kind'];\n if (typeof kind !== 'string' || !KINDS.includes(kind as StreamKind)) {\n throw new StreamProtocolError(`Unknown stream kind: ${String(kind)}.`);\n }\n if (expected !== undefined) {\n const allowed = typeof expected === 'string' ? [expected] : expected;\n if (!allowed.includes(kind as StreamKind)) {\n throw new StreamProtocolError(\n `Expected a ${allowed.join(' or ')} record, got ${kind}.`,\n );\n }\n }\n\n const agentId = record['agentId'];\n // Results and lifecycle events may concern an unassigned task, so only the\n // two directional streams require a producer identity.\n if (typeof agentId !== 'string') {\n throw new StreamProtocolError('Envelope has no agentId.');\n }\n if ((kind === 'task' || kind === 'report') && agentId.length === 0) {\n throw new StreamProtocolError('Envelope has no agentId.');\n }\n\n const workspaceId = record['workspaceId'];\n if (typeof workspaceId !== 'string' || workspaceId.length === 0) {\n throw new StreamProtocolError('Envelope has no workspaceId.');\n }\n\n const sentAt = record['sentAt'];\n if (typeof sentAt !== 'number' || !Number.isFinite(sentAt)) {\n throw new StreamProtocolError('Envelope has no sentAt.');\n }\n\n const payload = record['payload'];\n if (payload === null || typeof payload !== 'object') {\n throw new StreamProtocolError('Envelope has no payload.');\n }\n\n return {\n envelope: 1,\n kind: kind as StreamKind,\n agentId,\n workspaceId,\n sentAt,\n payload: payload as T,\n };\n}\n","import { Kafka, logLevel, type Consumer, type Producer, type SASLOptions } from 'kafkajs';\nimport type { StreamConfig } from './config.js';\nimport { unavailableError } from './config.js';\nimport { StreamProtocolError, StreamUnavailableError } from './errors.js';\nexport interface StreamLogger {\n info: (...args: unknown[]) => void;\n warn: (...args: unknown[]) => void;\n error: (...args: unknown[]) => void;\n debug: (...args: unknown[]) => void;\n}\n\n/** One record as consumers receive it. */\nexport interface StreamMessage {\n readonly key: string | null;\n readonly value: string;\n}\n\nexport interface StreamPublisher {\n connect(): Promise<void>;\n publish(topic: string, key: string, value: string): Promise<void>;\n /** Resolves once every accepted record has been acknowledged by the broker. */\n flush(): Promise<void>;\n disconnect(): Promise<void>;\n readonly publishedCount: number;\n readonly failedCount: number;\n}\n\nexport interface StreamSubscriber {\n connect(): Promise<void>;\n /**\n * Subscribes to `topics` and starts consuming. The handler is awaited, so a\n * throw leaves the offset uncommitted and the record is redelivered: every\n * handler must therefore be idempotent (the task service already is).\n */\n subscribe(\n topics: readonly string[],\n handler: (message: StreamMessage) => Promise<void>,\n ): Promise<void>;\n disconnect(): Promise<void>;\n readonly handledCount: number;\n readonly skippedCount: number;\n}\n\nfunction saslOptions(config: StreamConfig): SASLOptions | undefined {\n if (!config.sasl) return undefined;\n return {\n mechanism: config.sasl.mechanism,\n username: config.sasl.username,\n password: config.sasl.password,\n };\n}\n\nfunction createKafka(config: StreamConfig): Kafka {\n return new Kafka({\n clientId: config.clientId,\n brokers: [...config.brokers],\n ssl: config.ssl || undefined,\n sasl: saslOptions(config),\n connectionTimeout: config.connectionTimeoutMs,\n retry: { retries: config.retryCount, initialRetryTime: 300 },\n logLevel: logLevel.ERROR,\n });\n}\n\n/**\n * Creates the stream topics if the broker does not have them yet.\n *\n * Subscribing to a missing topic would make metadata refresh race the broker's\n * auto-creation, which surfaces as a transient `REPLICA_NOT_AVAILABLE` and\n * aborts startup. Creating the topics first, and waiting for their leaders, makes\n * the first subscribe deterministic on a fresh broker. Existing topics are left\n * untouched, including their partition count.\n *\n * Existing topics are filtered out before the create call: asking the broker to\n * create a topic that already exists is answered with a per-topic error, which\n * kafkajs reports at ERROR level. That noise on every restart trains operators\n * to ignore the stream logs, so the check is done here instead.\n */\nexport async function ensureTopics(\n config: StreamConfig,\n topics: readonly string[],\n logger?: StreamLogger,\n): Promise<void> {\n const admin = createKafka(config).admin();\n try {\n await admin.connect();\n } catch (error) {\n await admin.disconnect().catch(() => {});\n throw unavailableError(config, error);\n }\n\n try {\n const existing = new Set(await admin.listTopics());\n const missing = topics.filter((topic) => !existing.has(topic));\n if (missing.length === 0) return;\n\n const partitions = config.topicPartitions ?? 3;\n const replicationFactor = config.replicationFactor ?? 1;\n await admin.createTopics({\n topics: missing.map((topic) => ({\n topic,\n numPartitions: partitions,\n replicationFactor,\n })),\n waitForLeaders: true,\n });\n logger?.info(\n `Created stream topics: ${missing.join(', ')} ` +\n `(${partitions} partition(s), replication ${replicationFactor}).`,\n );\n } catch (error) {\n throw new StreamUnavailableError(\n `Could not create the stream topics (${topics.join(', ')}): ${(error as Error).message}`,\n config.brokers,\n error,\n );\n } finally {\n await admin.disconnect().catch(() => {});\n }\n}\n\n/**\n * Ordered, at-least-once producer.\n *\n * Sends are chained rather than fired in parallel so the broker sees one\n * agent's frames in the order they were produced. `acks: -1` means the promise\n * resolves only after every in-sync replica has the record, which is what makes\n * `flush()` a real guarantee before the process exits.\n */\nexport class KafkaPublisher implements StreamPublisher {\n private producer: Producer | null = null;\n private tail: Promise<void> = Promise.resolve();\n private sent = 0;\n private failed = 0;\n\n constructor(\n private readonly config: StreamConfig,\n private readonly logger?: StreamLogger,\n ) {}\n\n public get publishedCount(): number {\n return this.sent;\n }\n\n public get failedCount(): number {\n return this.failed;\n }\n\n public async connect(): Promise<void> {\n if (this.producer) return;\n const kafka = createKafka(this.config);\n try {\n const producer = kafka.producer({ allowAutoTopicCreation: true, idempotent: true });\n await producer.connect();\n this.producer = producer;\n } catch (error) {\n throw unavailableError(this.config, error);\n }\n this.logger?.debug(`Kafka producer connected to ${this.config.brokers.join(', ')}.`);\n }\n\n public publish(topic: string, key: string, value: string): Promise<void> {\n const producer = this.producer;\n if (!producer) {\n return Promise.reject(\n new StreamUnavailableError(\n 'The Kafka producer is not connected.',\n this.config.brokers,\n ),\n );\n }\n\n const send = this.tail.then(() =>\n producer.send({ topic, acks: -1, messages: [{ key, value }] }),\n );\n const settled = send.then(\n () => {\n this.sent += 1;\n },\n (error: unknown) => {\n this.failed += 1;\n this.logger?.error(\n `Could not publish to ${topic}: ${(error as Error).message}`,\n );\n throw error;\n },\n );\n // The chain must survive a failed send, or one broker hiccup would stall\n // every later record.\n this.tail = settled.catch(() => {});\n return settled;\n }\n\n public async flush(): Promise<void> {\n await this.tail;\n }\n\n public async disconnect(): Promise<void> {\n const producer = this.producer;\n this.producer = null;\n if (!producer) return;\n await this.tail.catch(() => {});\n await producer.disconnect().catch(() => {});\n }\n}\n\n/**\n * Consumer for one group.\n *\n * A poison record (bad envelope) is logged and skipped; anything else throws, so\n * the offset stays put and the record is retried. That is the only split where\n * at-least-once is safe: malformed input can never become authority, and valid\n * input is idempotent downstream.\n */\nexport class KafkaSubscriber implements StreamSubscriber {\n private consumer: Consumer | null = null;\n private started = false;\n private handled = 0;\n private skipped = 0;\n\n constructor(\n private readonly config: StreamConfig,\n private readonly groupId: string,\n private readonly options?: {\n readonly fromBeginning?: boolean;\n readonly logger?: StreamLogger;\n },\n ) {}\n\n public get handledCount(): number {\n return this.handled;\n }\n\n public get skippedCount(): number {\n return this.skipped;\n }\n\n public async connect(): Promise<void> {\n if (this.consumer) return;\n const kafka = createKafka(this.config);\n const consumer = kafka.consumer({\n groupId: this.groupId,\n sessionTimeout: 30_000,\n heartbeatInterval: 3_000,\n });\n try {\n await consumer.connect();\n } catch (error) {\n await consumer.disconnect().catch(() => {});\n throw unavailableError(this.config, error);\n }\n this.consumer = consumer;\n this.options?.logger?.debug(\n `Kafka consumer ${this.groupId} connected to ${this.config.brokers.join(', ')}.`,\n );\n }\n\n public async subscribe(\n topics: readonly string[],\n handler: (message: StreamMessage) => Promise<void>,\n ): Promise<void> {\n const consumer = this.consumer;\n if (!consumer) {\n throw new StreamUnavailableError(\n `Kafka consumer ${this.groupId} is not connected.`,\n this.config.brokers,\n );\n }\n if (this.started) {\n throw new Error(`Kafka consumer ${this.groupId} is already running.`);\n }\n\n const fromBeginning = this.options?.fromBeginning ?? false;\n for (const topic of topics) {\n await consumer.subscribe({ topic, fromBeginning });\n }\n\n await consumer.run({\n autoCommit: true,\n partitionsConsumedConcurrently: 1,\n eachMessage: async ({ topic, message }) => {\n if (!message.value) return;\n try {\n await handler({\n key: message.key ? message.key.toString() : null,\n value: message.value.toString(),\n });\n this.handled += 1;\n } catch (error) {\n if (error instanceof StreamProtocolError) {\n this.skipped += 1;\n this.options?.logger?.warn(\n `Skipping a malformed record on ${topic}: ${error.message}`,\n );\n return;\n }\n throw error;\n }\n },\n });\n this.started = true;\n }\n\n public async disconnect(): Promise<void> {\n const consumer = this.consumer;\n this.consumer = null;\n this.started = false;\n if (!consumer) return;\n await consumer.disconnect().catch(() => {});\n }\n}\n","/**\n * Topic names and partition keys.\n *\n * One topic per direction rather than one per agent: a topic per agent is\n * unbounded topic creation on a broker that is supposed to be boring. Ordering\n * is preserved per key instead, which is all the protocol needs:\n *\n * - commands are keyed by agent, so one agent's offers arrive in order;\n * - reports are keyed by task, so one task's pages, ack, progress and completion\n * arrive in order even when several agents report concurrently;\n * - results are keyed by task, for the same reason.\n */\nexport const STREAM_TOPICS = {\n /** Server to agent: offers, control frames, lease renewals, nacks. */\n commands: 'luxray.agent.commands',\n /**\n * Agent to server: page observations and acks/progress/checkpoints/completion.\n *\n * Pages share this topic with commands on purpose. Keyed by task, they land in\n * one partition, so the control plane applies every page before the report\n * that closes the same attempt. A `report` record always follows this order.\n */\n reports: 'luxray.agent.reports',\n /** Server to subscribers: durable lifecycle events, mirrored for consumers. */\n events: 'luxray.tasks.events',\n /** Server to subscribers: page observations, mirrored after they persist. */\n results: 'luxray.crawl.results',\n} as const;\n\nexport type StreamTopic = (typeof STREAM_TOPICS)[keyof typeof STREAM_TOPICS];\n\n/**\n * Every stream topic, in the order a fresh broker should be given them.\n *\n * Consumers create topics explicitly before subscribing: subscribing to a topic\n * that does not exist yet makes metadata refresh race topic creation, and a\n * control plane must not fail to start on that transient.\n */\nexport const STREAM_TOPIC_LIST: readonly StreamTopic[] = [\n STREAM_TOPICS.commands,\n STREAM_TOPICS.reports,\n STREAM_TOPICS.results,\n STREAM_TOPICS.events,\n];\n\n/** Each agent is its own consumer group, so it sees only its own commands. */\nexport function agentConsumerGroup(agentId: string): string {\n return `luxray-agent-${agentId}`;\n}\n\n/** All API instances share a group: a report is applied exactly once. */\nexport const API_REPORTS_GROUP = 'luxray-api-reports';\nexport const API_RESULTS_GROUP = 'luxray-api-results';\n","import type {\n AgentTaskCancelMessage,\n AgentTaskClientMessage,\n AgentTaskNackMessage,\n AgentTaskOfferMessage,\n AgentTaskPauseMessage,\n AgentTaskRenewedMessage,\n AgentTaskServerMessage,\n CrawlPageResult,\n StreamEnvelope,\n} from '@luxray/shared';\nimport {\n KafkaPublisher,\n KafkaSubscriber,\n STREAM_TOPICS,\n agentConsumerGroup,\n decodeEnvelope,\n encodeEnvelope,\n ensureTopics,\n type StreamConfig,\n type StreamMessage,\n type StreamPublisher,\n type StreamSubscriber,\n} from '@luxray/streaming';\n\nexport interface TaskStreamLogger {\n info: (...args: unknown[]) => void;\n warn: (...args: unknown[]) => void;\n error: (...args: unknown[]) => void;\n debug: (...args: unknown[]) => void;\n}\n\n/**\n * Receives the commands addressed to this agent.\n *\n * Handlers are synchronous by design: the executor's decisions (ack, run, stop)\n * are decisions about local state, and a Kafka handler that awaits a long job\n * would stall the partition behind it.\n */\nexport interface TaskStreamHandlers {\n readonly onOffer: (message: AgentTaskOfferMessage) => void;\n readonly onCancel: (message: AgentTaskCancelMessage) => void;\n readonly onPause: (message: AgentTaskPauseMessage) => void;\n readonly onRenewed: (message: AgentTaskRenewedMessage) => void;\n readonly onNack: (message: AgentTaskNackMessage) => void;\n}\n\nexport interface TaskStreamOptions {\n readonly config: StreamConfig;\n readonly agentId: string;\n readonly workspaceId: string;\n readonly handlers: TaskStreamHandlers;\n readonly logger?: TaskStreamLogger;\n /** Injected in tests so the stream can be exercised without a broker. */\n readonly publisher?: StreamPublisher;\n readonly subscriber?: StreamSubscriber;\n /** Injected in tests; the default creates the topics on the real broker. */\n readonly ensureTopics?: (config: StreamConfig) => Promise<void>;\n}\n\n/**\n * The agent's side of the Kafka transport.\n *\n * Reports and page results leave through the publisher; commands arrive on a\n * consumer group private to this agent. The group is what makes an offer\n * durable: a command produced while the agent is offline is still there when it\n * reconnects, whereas the old socket could only replay what the control plane\n * remembered to resend.\n */\nexport class TaskStream {\n private readonly publisher: StreamPublisher;\n private readonly subscriber: StreamSubscriber;\n private readonly handlers: TaskStreamHandlers;\n private readonly logger: TaskStreamLogger;\n private readonly prepareTopics: (config: StreamConfig) => Promise<void>;\n private connected = false;\n\n constructor(private readonly options: TaskStreamOptions) {\n this.publisher =\n options.publisher ?? new KafkaPublisher(options.config, options.logger);\n this.subscriber =\n options.subscriber ??\n new KafkaSubscriber(options.config, agentConsumerGroup(options.agentId), {\n fromBeginning: true,\n logger: options.logger,\n });\n this.handlers = options.handlers;\n this.prepareTopics =\n options.ensureTopics ??\n ((config) =>\n ensureTopics(config, [\n STREAM_TOPICS.commands,\n STREAM_TOPICS.reports,\n STREAM_TOPICS.results,\n ]));\n this.logger = options.logger ?? {\n info: () => {},\n warn: (...args: unknown[]) => console.warn(...args),\n error: (...args: unknown[]) => console.error(...args),\n debug: () => {},\n };\n }\n\n public get isConnected(): boolean {\n return this.connected;\n }\n\n public get publishedCount(): number {\n return this.publisher.publishedCount;\n }\n\n /**\n * Connects both sides before anything is produced.\n *\n * The subscriber connects first in spirit as well as in order: a command\n * published between \"agent is online\" and \"agent is consuming\" would still be\n * delivered later, but a producer that is not connected silently rejects\n * every report it is handed.\n */\n public async connect(): Promise<void> {\n if (this.connected) return;\n await this.subscriber.connect();\n try {\n await this.publisher.connect();\n // A fresh broker has no topics; creating them before subscribing avoids\n // the metadata race that would otherwise look like an unavailable broker.\n await this.prepareTopics(this.options.config);\n await this.subscriber.subscribe([STREAM_TOPICS.commands], (message) =>\n this.receive(message),\n );\n } catch (error) {\n await this.subscriber.disconnect().catch(() => {});\n await this.publisher.disconnect().catch(() => {});\n throw error;\n }\n this.connected = true;\n }\n\n /**\n * Publishes one execution report.\n *\n * Returns false when the stream is not connected, which the executor already\n * treats as a transient gap. A publish failure after acceptance is logged,\n * not thrown: the lease will expire and the control plane will reclaim the\n * task, which is the same outcome as a socket write that never arrived.\n */\n public report(message: AgentTaskClientMessage): boolean {\n if (!this.connected) return false;\n\n const envelope: StreamEnvelope<AgentTaskClientMessage> = {\n envelope: 1,\n kind: 'report',\n agentId: this.options.agentId,\n workspaceId: this.options.workspaceId,\n sentAt: Date.now(),\n payload: message,\n };\n\n // Keyed by task: one task's ack/progress/complete stay in order even when\n // several jobs on this agent publish at once.\n void this.publisher\n .publish(STREAM_TOPICS.reports, message.taskId, encodeEnvelope(envelope))\n .catch((error: unknown) => {\n this.logger.error(\n `Could not publish ${message.type} for task ${message.taskId}: ${(error as Error).message}`,\n );\n });\n return true;\n }\n\n /**\n * Streams one observed page while the crawl is still running.\n *\n * Pages share the topic with reports and are keyed by task, so the completion\n * that follows them lands in the same partition and can never overtake them.\n */\n public publishPage(page: CrawlPageResult): boolean {\n if (!this.connected) return false;\n\n const envelope: StreamEnvelope<CrawlPageResult> = {\n envelope: 1,\n kind: 'result',\n agentId: this.options.agentId,\n workspaceId: this.options.workspaceId,\n sentAt: Date.now(),\n payload: page,\n };\n\n void this.publisher\n .publish(STREAM_TOPICS.reports, page.taskId, encodeEnvelope(envelope))\n .catch((error: unknown) => {\n this.logger.warn(\n `Could not publish page ${page.sequence} of task ${page.taskId}: ${(error as Error).message}`,\n );\n });\n return true;\n }\n\n /** Waits until every accepted record was acknowledged by the broker. */\n public async flush(): Promise<void> {\n await this.publisher.flush();\n }\n\n public async close(): Promise<void> {\n if (!this.connected) {\n await this.publisher.disconnect().catch(() => {});\n await this.subscriber.disconnect().catch(() => {});\n return;\n }\n this.connected = false;\n // Flush before disconnecting, or a completion queued during shutdown is\n // dropped and the task sits RUNNING until its lease expires.\n await this.flush().catch((error: unknown) => {\n this.logger.error(`Could not flush the task stream: ${(error as Error).message}`);\n });\n await this.subscriber.disconnect();\n await this.publisher.disconnect();\n }\n\n private async receive(message: StreamMessage): Promise<void> {\n const envelope = decodeEnvelope<AgentTaskServerMessage>(message.value, 'task');\n\n // The group is private to this agent, so a foreign envelope means a\n // misconfigured producer, not a message for us. Ignoring it is safer than\n // executing a task this agent was never assigned.\n if (envelope.agentId !== this.options.agentId) {\n this.logger.warn(\n `Ignoring a ${envelope.payload.type} record addressed to agent ${envelope.agentId}.`,\n );\n return;\n }\n\n const payload = envelope.payload;\n switch (payload.type) {\n case 'task.offer':\n this.handlers.onOffer(payload);\n return;\n case 'task.cancel':\n this.handlers.onCancel(payload);\n return;\n case 'task.pause':\n this.handlers.onPause(payload);\n return;\n case 'task.renewed':\n this.handlers.onRenewed(payload);\n return;\n case 'task.nack':\n this.handlers.onNack(payload);\n return;\n default: {\n // A newer server may send a command this build does not know. Retrying\n // it would block every later command on this agent's partition, and it\n // can never succeed, so it is logged and dropped. The durable task row\n // is unaffected; the lease sweep is the backstop.\n const unknown = payload as { type?: unknown };\n this.logger.warn(`Ignoring an unknown task command: ${String(unknown.type)}.`);\n return;\n }\n }\n }\n}\n"],"mappings":";AACM,IAAO,yBAAP,cAAsC,MAAK;EAK7B;EACS;EALF,OAAO;EAEhC,YACE,SACgB,SACS,OAAe;AAExC,UAAM,OAAO;AAHG,SAAA,UAAA;AACS,SAAA,QAAA;EAG3B;;AAII,IAAO,sBAAP,cAAmC,MAAK;EACnB,OAAO;;;;ACqE5B,SAAU,iBAAiB,QAAsB,OAAc;AACnE,QAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,SAAO,IAAI,uBACT,6BAA6B,OAAO,QAAQ,KAAK,IAAI,CAAC,KAAK,MAAM,wFAEjE,OAAO,SACP,KAAK;AAET;;;ACzFA,IAAM,QAA+B,CAAC,QAAQ,UAAU,SAAS,QAAQ;AAEnE,SAAU,eAAkB,UAA2B;AAC3D,SAAO,KAAK,UAAU,QAAQ;AAChC;AAaM,SAAU,eACd,KACA,UAA6C;AAE7C,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;EACzB,SAAS,OAAO;AACd,UAAM,IAAI,oBAAoB,uBAAwB,MAAgB,OAAO,EAAE;EACjF;AAEA,MAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GAAG;AAC1E,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,QAAM,SAAS;AACf,MAAI,OAAO,UAAU,MAAM,GAAG;AAC5B,UAAM,IAAI,oBAAoB,iCAAiC,OAAO,OAAO,UAAU,CAAC,CAAC,GAAG;EAC9F;AAEA,QAAM,OAAO,OAAO,MAAM;AAC1B,MAAI,OAAO,SAAS,YAAY,CAAC,MAAM,SAAS,IAAkB,GAAG;AACnE,UAAM,IAAI,oBAAoB,wBAAwB,OAAO,IAAI,CAAC,GAAG;EACvE;AACA,MAAI,aAAa,QAAW;AAC1B,UAAM,UAAU,OAAO,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC5D,QAAI,CAAC,QAAQ,SAAS,IAAkB,GAAG;AACzC,YAAM,IAAI,oBACR,cAAc,QAAQ,KAAK,MAAM,CAAC,gBAAgB,IAAI,GAAG;IAE7D;EACF;AAEA,QAAM,UAAU,OAAO,SAAS;AAGhC,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AACA,OAAK,SAAS,UAAU,SAAS,aAAa,QAAQ,WAAW,GAAG;AAClE,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,QAAM,cAAc,OAAO,aAAa;AACxC,MAAI,OAAO,gBAAgB,YAAY,YAAY,WAAW,GAAG;AAC/D,UAAM,IAAI,oBAAoB,8BAA8B;EAC9D;AAEA,QAAM,SAAS,OAAO,QAAQ;AAC9B,MAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,MAAM,GAAG;AAC1D,UAAM,IAAI,oBAAoB,yBAAyB;EACzD;AAEA,QAAM,UAAU,OAAO,SAAS;AAChC,MAAI,YAAY,QAAQ,OAAO,YAAY,UAAU;AACnD,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,SAAO;IACL,UAAU;IACV;IACA;IACA;IACA;IACA;;AAEJ;;;ACtFA,SAAS,OAAO,gBAAgE;AA2ChF,SAAS,YAAY,QAAoB;AACvC,MAAI,CAAC,OAAO;AAAM,WAAO;AACzB,SAAO;IACL,WAAW,OAAO,KAAK;IACvB,UAAU,OAAO,KAAK;IACtB,UAAU,OAAO,KAAK;;AAE1B;AAEA,SAAS,YAAY,QAAoB;AACvC,SAAO,IAAI,MAAM;IACf,UAAU,OAAO;IACjB,SAAS,CAAC,GAAG,OAAO,OAAO;IAC3B,KAAK,OAAO,OAAO;IACnB,MAAM,YAAY,MAAM;IACxB,mBAAmB,OAAO;IAC1B,OAAO,EAAE,SAAS,OAAO,YAAY,kBAAkB,IAAG;IAC1D,UAAU,SAAS;GACpB;AACH;AAgBA,eAAsB,aACpB,QACA,QACA,QAAqB;AAErB,QAAM,QAAQ,YAAY,MAAM,EAAE,MAAK;AACvC,MAAI;AACF,UAAM,MAAM,QAAO;EACrB,SAAS,OAAO;AACd,UAAM,MAAM,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;AACvC,UAAM,iBAAiB,QAAQ,KAAK;EACtC;AAEA,MAAI;AACF,UAAM,WAAW,IAAI,IAAI,MAAM,MAAM,WAAU,CAAE;AACjD,UAAM,UAAU,OAAO,OAAO,CAAC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC;AAC7D,QAAI,QAAQ,WAAW;AAAG;AAE1B,UAAM,aAAa,OAAO,mBAAmB;AAC7C,UAAM,oBAAoB,OAAO,qBAAqB;AACtD,UAAM,MAAM,aAAa;MACvB,QAAQ,QAAQ,IAAI,CAAC,WAAW;QAC9B;QACA,eAAe;QACf;QACA;MACF,gBAAgB;KACjB;AACD,YAAQ,KACN,0BAA0B,QAAQ,KAAK,IAAI,CAAC,KACtC,UAAU,8BAA8B,iBAAiB,IAAI;EAEvE,SAAS,OAAO;AACd,UAAM,IAAI,uBACR,uCAAuC,OAAO,KAAK,IAAI,CAAC,MAAO,MAAgB,OAAO,IACtF,OAAO,SACP,KAAK;EAET;AACE,UAAM,MAAM,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EACzC;AACF;AAUM,IAAO,iBAAP,MAAqB;EAON;EACA;EAPX,WAA4B;EAC5B,OAAsB,QAAQ,QAAO;EACrC,OAAO;EACP,SAAS;EAEjB,YACmB,QACA,QAAqB;AADrB,SAAA,SAAA;AACA,SAAA,SAAA;EAChB;EAEH,IAAW,iBAAc;AACvB,WAAO,KAAK;EACd;EAEA,IAAW,cAAW;AACpB,WAAO,KAAK;EACd;EAEO,MAAM,UAAO;AAClB,QAAI,KAAK;AAAU;AACnB,UAAM,QAAQ,YAAY,KAAK,MAAM;AACrC,QAAI;AACF,YAAM,WAAW,MAAM,SAAS,EAAE,wBAAwB,MAAM,YAAY,KAAI,CAAE;AAClF,YAAM,SAAS,QAAO;AACtB,WAAK,WAAW;IAClB,SAAS,OAAO;AACd,YAAM,iBAAiB,KAAK,QAAQ,KAAK;IAC3C;AACA,SAAK,QAAQ,MAAM,+BAA+B,KAAK,OAAO,QAAQ,KAAK,IAAI,CAAC,GAAG;EACrF;EAEO,QAAQ,OAAe,KAAa,OAAa;AACtD,UAAM,WAAW,KAAK;AACtB,QAAI,CAAC,UAAU;AACb,aAAO,QAAQ,OACb,IAAI,uBACF,wCACA,KAAK,OAAO,OAAO,CACpB;IAEL;AAEA,UAAM,OAAO,KAAK,KAAK,KAAK,MAC1B,SAAS,KAAK,EAAE,OAAO,MAAM,IAAI,UAAU,CAAC,EAAE,KAAK,MAAK,CAAE,EAAC,CAAE,CAAC;AAEhE,UAAM,UAAU,KAAK,KACnB,MAAK;AACH,WAAK,QAAQ;IACf,GACA,CAAC,UAAkB;AACjB,WAAK,UAAU;AACf,WAAK,QAAQ,MACX,wBAAwB,KAAK,KAAM,MAAgB,OAAO,EAAE;AAE9D,YAAM;IACR,CAAC;AAIH,SAAK,OAAO,QAAQ,MAAM,MAAK;IAAE,CAAC;AAClC,WAAO;EACT;EAEO,MAAM,QAAK;AAChB,UAAM,KAAK;EACb;EAEO,MAAM,aAAU;AACrB,UAAM,WAAW,KAAK;AACtB,SAAK,WAAW;AAChB,QAAI,CAAC;AAAU;AACf,UAAM,KAAK,KAAK,MAAM,MAAK;IAAE,CAAC;AAC9B,UAAM,SAAS,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EAC5C;;AAWI,IAAO,kBAAP,MAAsB;EAOP;EACA;EACA;EARX,WAA4B;EAC5B,UAAU;EACV,UAAU;EACV,UAAU;EAElB,YACmB,QACA,SACA,SAGhB;AALgB,SAAA,SAAA;AACA,SAAA,UAAA;AACA,SAAA,UAAA;EAIhB;EAEH,IAAW,eAAY;AACrB,WAAO,KAAK;EACd;EAEA,IAAW,eAAY;AACrB,WAAO,KAAK;EACd;EAEO,MAAM,UAAO;AAClB,QAAI,KAAK;AAAU;AACnB,UAAM,QAAQ,YAAY,KAAK,MAAM;AACrC,UAAM,WAAW,MAAM,SAAS;MAC9B,SAAS,KAAK;MACd,gBAAgB;MAChB,mBAAmB;KACpB;AACD,QAAI;AACF,YAAM,SAAS,QAAO;IACxB,SAAS,OAAO;AACd,YAAM,SAAS,WAAU,EAAG,MAAM,MAAK;MAAE,CAAC;AAC1C,YAAM,iBAAiB,KAAK,QAAQ,KAAK;IAC3C;AACA,SAAK,WAAW;AAChB,SAAK,SAAS,QAAQ,MACpB,kBAAkB,KAAK,OAAO,iBAAiB,KAAK,OAAO,QAAQ,KAAK,IAAI,CAAC,GAAG;EAEpF;EAEO,MAAM,UACX,QACA,SAAkD;AAElD,UAAM,WAAW,KAAK;AACtB,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,uBACR,kBAAkB,KAAK,OAAO,sBAC9B,KAAK,OAAO,OAAO;IAEvB;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB,KAAK,OAAO,sBAAsB;IACtE;AAEA,UAAM,gBAAgB,KAAK,SAAS,iBAAiB;AACrD,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,UAAU,EAAE,OAAO,cAAa,CAAE;IACnD;AAEA,UAAM,SAAS,IAAI;MACjB,YAAY;MACZ,gCAAgC;MAChC,aAAa,OAAO,EAAE,OAAO,QAAO,MAAM;AACxC,YAAI,CAAC,QAAQ;AAAO;AACpB,YAAI;AACF,gBAAM,QAAQ;YACZ,KAAK,QAAQ,MAAM,QAAQ,IAAI,SAAQ,IAAK;YAC5C,OAAO,QAAQ,MAAM,SAAQ;WAC9B;AACD,eAAK,WAAW;QAClB,SAAS,OAAO;AACd,cAAI,iBAAiB,qBAAqB;AACxC,iBAAK,WAAW;AAChB,iBAAK,SAAS,QAAQ,KACpB,kCAAkC,KAAK,KAAK,MAAM,OAAO,EAAE;AAE7D;UACF;AACA,gBAAM;QACR;MACF;KACD;AACD,SAAK,UAAU;EACjB;EAEO,MAAM,aAAU;AACrB,UAAM,WAAW,KAAK;AACtB,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,QAAI,CAAC;AAAU;AACf,UAAM,SAAS,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EAC5C;;;;ACzSK,IAAM,gBAAgB;;EAE3B,UAAU;;;;;;;;EAQV,SAAS;;EAET,QAAQ;;EAER,SAAS;;AAYJ,IAAM,oBAA4C;EACvD,cAAc;EACd,cAAc;EACd,cAAc;EACd,cAAc;;AAIV,SAAU,mBAAmB,SAAe;AAChD,SAAO,gBAAgB,OAAO;AAChC;;;ACqBO,IAAM,aAAN,MAAiB;AAAA,EAQtB,YAA6B,SAA4B;AAA5B;AAC3B,SAAK,YACH,QAAQ,aAAa,IAAI,eAAe,QAAQ,QAAQ,QAAQ,MAAM;AACxE,SAAK,aACH,QAAQ,cACR,IAAI,gBAAgB,QAAQ,QAAQ,mBAAmB,QAAQ,OAAO,GAAG;AAAA,MACvE,eAAe;AAAA,MACf,QAAQ,QAAQ;AAAA,IAClB,CAAC;AACH,SAAK,WAAW,QAAQ;AACxB,SAAK,gBACH,QAAQ,iBACP,CAAC,WACA,aAAa,QAAQ;AAAA,MACnB,cAAc;AAAA,MACd,cAAc;AAAA,MACd,cAAc;AAAA,IAChB,CAAC;AACL,SAAK,SAAS,QAAQ,UAAU;AAAA,MAC9B,MAAM,MAAM;AAAA,MAAC;AAAA,MACb,MAAM,IAAI,SAAoB,QAAQ,KAAK,GAAG,IAAI;AAAA,MAClD,OAAO,IAAI,SAAoB,QAAQ,MAAM,GAAG,IAAI;AAAA,MACpD,OAAO,MAAM;AAAA,MAAC;AAAA,IAChB;AAAA,EACF;AAAA,EAxB6B;AAAA,EAPZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT,YAAY;AAAA,EA4BpB,IAAW,cAAuB;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAW,iBAAyB;AAClC,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAa,UAAyB;AACpC,QAAI,KAAK,UAAW;AACpB,UAAM,KAAK,WAAW,QAAQ;AAC9B,QAAI;AACF,YAAM,KAAK,UAAU,QAAQ;AAG7B,YAAM,KAAK,cAAc,KAAK,QAAQ,MAAM;AAC5C,YAAM,KAAK,WAAW;AAAA,QAAU,CAAC,cAAc,QAAQ;AAAA,QAAG,CAAC,YACzD,KAAK,QAAQ,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,YAAM,KAAK,WAAW,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACjD,YAAM,KAAK,UAAU,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAChD,YAAM;AAAA,IACR;AACA,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUO,OAAO,SAA0C;AACtD,QAAI,CAAC,KAAK,UAAW,QAAO;AAE5B,UAAM,WAAmD;AAAA,MACvD,UAAU;AAAA,MACV,MAAM;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,aAAa,KAAK,QAAQ;AAAA,MAC1B,QAAQ,KAAK,IAAI;AAAA,MACjB,SAAS;AAAA,IACX;AAIA,SAAK,KAAK,UACP,QAAQ,cAAc,SAAS,QAAQ,QAAQ,eAAe,QAAQ,CAAC,EACvE,MAAM,CAAC,UAAmB;AACzB,WAAK,OAAO;AAAA,QACV,qBAAqB,QAAQ,IAAI,aAAa,QAAQ,MAAM,KAAM,MAAgB,OAAO;AAAA,MAC3F;AAAA,IACF,CAAC;AACH,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,YAAY,MAAgC;AACjD,QAAI,CAAC,KAAK,UAAW,QAAO;AAE5B,UAAM,WAA4C;AAAA,MAChD,UAAU;AAAA,MACV,MAAM;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,aAAa,KAAK,QAAQ;AAAA,MAC1B,QAAQ,KAAK,IAAI;AAAA,MACjB,SAAS;AAAA,IACX;AAEA,SAAK,KAAK,UACP,QAAQ,cAAc,SAAS,KAAK,QAAQ,eAAe,QAAQ,CAAC,EACpE,MAAM,CAAC,UAAmB;AACzB,WAAK,OAAO;AAAA,QACV,0BAA0B,KAAK,QAAQ,YAAY,KAAK,MAAM,KAAM,MAAgB,OAAO;AAAA,MAC7F;AAAA,IACF,CAAC;AACH,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAa,QAAuB;AAClC,UAAM,KAAK,UAAU,MAAM;AAAA,EAC7B;AAAA,EAEA,MAAa,QAAuB;AAClC,QAAI,CAAC,KAAK,WAAW;AACnB,YAAM,KAAK,UAAU,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAChD,YAAM,KAAK,WAAW,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACjD;AAAA,IACF;AACA,SAAK,YAAY;AAGjB,UAAM,KAAK,MAAM,EAAE,MAAM,CAAC,UAAmB;AAC3C,WAAK,OAAO,MAAM,oCAAqC,MAAgB,OAAO,EAAE;AAAA,IAClF,CAAC;AACD,UAAM,KAAK,WAAW,WAAW;AACjC,UAAM,KAAK,UAAU,WAAW;AAAA,EAClC;AAAA,EAEA,MAAc,QAAQ,SAAuC;AAC3D,UAAM,WAAW,eAAuC,QAAQ,OAAO,MAAM;AAK7E,QAAI,SAAS,YAAY,KAAK,QAAQ,SAAS;AAC7C,WAAK,OAAO;AAAA,QACV,cAAc,SAAS,QAAQ,IAAI,8BAA8B,SAAS,OAAO;AAAA,MACnF;AACA;AAAA,IACF;AAEA,UAAM,UAAU,SAAS;AACzB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,aAAK,SAAS,QAAQ,OAAO;AAC7B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,SAAS,OAAO;AAC9B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,QAAQ,OAAO;AAC7B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,UAAU,OAAO;AAC/B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,OAAO,OAAO;AAC5B;AAAA,MACF,SAAS;AAKP,cAAM,UAAU;AAChB,aAAK,OAAO,KAAK,qCAAqC,OAAO,QAAQ,IAAI,CAAC,GAAG;AAC7E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":[]}
1
+ {"version":3,"sources":["../../../streaming/src/config.ts","../../../streaming/src/errors.ts","../../../streaming/src/envelope.ts","../../../streaming/src/kafka.ts","../../../streaming/src/topics.ts","../../src/client/task-stream.ts"],"sourcesContent":["import { existsSync, readFileSync, readdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ConnectionOptions } from 'node:tls';\nimport { StreamUnavailableError } from './errors.js';\n\nexport interface StreamSaslConfig {\n readonly mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';\n readonly username: string;\n readonly password: string;\n}\n\nexport interface StreamConfig {\n readonly brokers: readonly string[];\n readonly clientId: string;\n readonly ssl: boolean | ConnectionOptions;\n readonly sasl: StreamSaslConfig | null;\n /** How long a broker connection may take before it counts as unavailable. */\n readonly connectionTimeoutMs: number;\n /** Retries per connect attempt; kept small so a missing broker fails fast. */\n readonly retryCount: number;\n /** Partitions used when a stream topic has to be created. Default 3. */\n readonly topicPartitions?: number;\n /** Replication factor used when a stream topic has to be created. Default 1. */\n readonly replicationFactor?: number;\n}\n\nconst DEFAULT_BROKERS = '127.0.0.1:9092';\n\nfunction findRootCert(): string | undefined {\n try {\n const cwd = process.cwd();\n const commonNames = [\n 'cl.pem',\n 'cl.cert',\n 'cl.crt',\n 'ca.pem',\n 'ca.cert',\n 'ca.crt',\n 'certificate.pem',\n 'certificate.crt',\n ];\n for (const name of commonNames) {\n const fullPath = join(cwd, name);\n if (existsSync(fullPath)) {\n return readFileSync(fullPath, 'utf-8');\n }\n }\n const files = readdirSync(cwd);\n for (const file of files) {\n const lower = file.toLowerCase();\n if (\n (lower.includes('cl') || lower.includes('ca') || lower.includes('cert')) &&\n (lower.endsWith('.pem') || lower.endsWith('.crt') || lower.endsWith('.cert'))\n ) {\n return readFileSync(join(cwd, file), 'utf-8');\n }\n }\n } catch {\n // Ignore filesystem read errors\n }\n return undefined;\n}\n\nfunction resolveSsl(\n env: NodeJS.ProcessEnv,\n brokers: readonly string[],\n hasSasl: boolean,\n): boolean | ConnectionOptions {\n if (env['KAFKA_SSL']?.toLowerCase() === 'false') {\n return false;\n }\n\n const cert = findRootCert();\n const explicitSsl = env['KAFKA_SSL']?.toLowerCase() === 'true';\n const isRemote = brokers.some(\n (b) => !b.startsWith('127.0.0.1') && !b.startsWith('localhost'),\n );\n\n const shouldEnable = explicitSsl || hasSasl || !!cert || isRemote;\n if (!shouldEnable) {\n return false;\n }\n\n if (cert) {\n return {\n ca: [cert],\n rejectUnauthorized: true,\n };\n }\n\n return true;\n}\n\nexport function streamConfigFromEnv(\n env: NodeJS.ProcessEnv = process.env,\n overrides: Partial<StreamConfig> = {},\n): StreamConfig {\n const rawBrokers = env['KAFKA_BROKERS'] ?? env['KAFKA_HOST'] ?? DEFAULT_BROKERS;\n const brokers = rawBrokers\n .split(',')\n .map((broker) => broker.trim())\n .filter((broker) => broker.length > 0);\n\n if (brokers.length === 0) {\n throw new Error('KAFKA_BROKERS is set but lists no broker address.');\n }\n\n const sasl = saslFromEnv(env);\n const ssl = overrides.ssl !== undefined ? overrides.ssl : resolveSsl(env, brokers, sasl !== null);\n\n const merged: StreamConfig = {\n brokers,\n clientId: env['KAFKA_CLIENT_ID'] ?? 'luxray',\n ssl,\n sasl,\n connectionTimeoutMs: 5_000,\n retryCount: 2,\n topicPartitions: positiveInt(env['KAFKA_TOPIC_PARTITIONS'], 3, 'KAFKA_TOPIC_PARTITIONS'),\n replicationFactor: positiveInt(\n env['KAFKA_REPLICATION_FACTOR'],\n 1,\n 'KAFKA_REPLICATION_FACTOR',\n ),\n ...overrides,\n };\n\n // A retry count of zero is rejected by kafkajs whenever the producer is\n // idempotent, which it always is here.\n return { ...merged, retryCount: Math.max(1, merged.retryCount) };\n}\n\nfunction saslFromEnv(env: NodeJS.ProcessEnv): StreamSaslConfig | null {\n const username = env['KAFKA_SASL_USERNAME'] ?? env['KAFKA_USER'] ?? env['KAFKA_USERNAME'];\n const password = env['KAFKA_SASL_PASSWORD'] ?? env['KAFKA_PASS'] ?? env['KAFKA_PASSWORD'];\n if (!username || !password) return null;\n\n const raw = (env['KAFKA_SASL_MECHANISM'] ?? 'scram-sha-512').toLowerCase();\n if (raw !== 'plain' && raw !== 'scram-sha-256' && raw !== 'scram-sha-512') {\n throw new Error(\n `KAFKA_SASL_MECHANISM must be plain, scram-sha-256 or scram-sha-512 (got ${raw}).`,\n );\n }\n return { mechanism: raw, username, password };\n}\n\n/** Parses a positive integer setting, naming the variable in the error. */\nfunction positiveInt(raw: string | undefined, fallback: number, name: string): number {\n if (raw === undefined || raw.trim() === '') return fallback;\n const value = Number(raw);\n if (!Number.isInteger(value) || value < 1) {\n throw new Error(`${name} must be a positive integer (got \"${raw}\").`);\n }\n return value;\n}\n\n/** One error message for both processes, naming the thing to fix. */\nexport function unavailableError(config: StreamConfig, cause: unknown): StreamUnavailableError {\n const detail = cause instanceof Error ? cause.message : String(cause);\n return new StreamUnavailableError(\n `Kafka is not reachable at ${config.brokers.join(', ')} (${detail}). ` +\n 'Start it with `docker compose up -d kafka`, or point KAFKA_BROKERS at a broker.',\n config.brokers,\n cause,\n );\n}\n","/** Raised when the broker cannot be reached or refuses the connection. */\nexport class StreamUnavailableError extends Error {\n public override readonly name = 'StreamUnavailableError';\n\n constructor(\n message: string,\n public readonly brokers: readonly string[],\n public override readonly cause?: unknown,\n ) {\n super(message);\n }\n}\n\n/** Raised when a record is not a valid envelope. Never inferred as meaning. */\nexport class StreamProtocolError extends Error {\n public override readonly name = 'StreamProtocolError';\n}\n","import type { StreamEnvelope, StreamKind } from '@luxray/shared';\nimport { StreamProtocolError } from './errors.js';\n\nconst KINDS: readonly StreamKind[] = ['task', 'report', 'event', 'result'];\n\nexport function encodeEnvelope<T>(envelope: StreamEnvelope<T>): string {\n return JSON.stringify(envelope);\n}\n\n/**\n * Parses one record.\n *\n * A malformed record is a protocol error, never a partially usable message:\n * acting on half an offer would be worse than dropping it, because the durable\n * offer stays in the task row and is replayed on the next connection.\n *\n * `expected` may name several kinds when a topic deliberately carries more than\n * one. Page results share the report topic so a task's pages and its completion\n * stay in one partition order; the caller dispatches on the returned kind.\n */\nexport function decodeEnvelope<T>(\n raw: string,\n expected?: StreamKind | readonly StreamKind[],\n): StreamEnvelope<T> {\n let parsed: unknown;\n try {\n parsed = JSON.parse(raw);\n } catch (error) {\n throw new StreamProtocolError(`Record is not JSON: ${(error as Error).message}`);\n }\n\n if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {\n throw new StreamProtocolError('Record is not an object.');\n }\n\n const record = parsed as Record<string, unknown>;\n if (record['envelope'] !== 1) {\n throw new StreamProtocolError(`Unsupported envelope version: ${String(record['envelope'])}.`);\n }\n\n const kind = record['kind'];\n if (typeof kind !== 'string' || !KINDS.includes(kind as StreamKind)) {\n throw new StreamProtocolError(`Unknown stream kind: ${String(kind)}.`);\n }\n if (expected !== undefined) {\n const allowed = typeof expected === 'string' ? [expected] : expected;\n if (!allowed.includes(kind as StreamKind)) {\n throw new StreamProtocolError(\n `Expected a ${allowed.join(' or ')} record, got ${kind}.`,\n );\n }\n }\n\n const agentId = record['agentId'];\n // Results and lifecycle events may concern an unassigned task, so only the\n // two directional streams require a producer identity.\n if (typeof agentId !== 'string') {\n throw new StreamProtocolError('Envelope has no agentId.');\n }\n if ((kind === 'task' || kind === 'report') && agentId.length === 0) {\n throw new StreamProtocolError('Envelope has no agentId.');\n }\n\n const workspaceId = record['workspaceId'];\n if (typeof workspaceId !== 'string' || workspaceId.length === 0) {\n throw new StreamProtocolError('Envelope has no workspaceId.');\n }\n\n const sentAt = record['sentAt'];\n if (typeof sentAt !== 'number' || !Number.isFinite(sentAt)) {\n throw new StreamProtocolError('Envelope has no sentAt.');\n }\n\n const payload = record['payload'];\n if (payload === null || typeof payload !== 'object') {\n throw new StreamProtocolError('Envelope has no payload.');\n }\n\n return {\n envelope: 1,\n kind: kind as StreamKind,\n agentId,\n workspaceId,\n sentAt,\n payload: payload as T,\n };\n}\n","import { Kafka, logLevel, type Consumer, type Producer, type SASLOptions } from 'kafkajs';\nimport type { StreamConfig } from './config.js';\nimport { unavailableError } from './config.js';\nimport { StreamProtocolError, StreamUnavailableError } from './errors.js';\nexport interface StreamLogger {\n info: (...args: unknown[]) => void;\n warn: (...args: unknown[]) => void;\n error: (...args: unknown[]) => void;\n debug: (...args: unknown[]) => void;\n}\n\n/** One record as consumers receive it. */\nexport interface StreamMessage {\n readonly key: string | null;\n readonly value: string;\n}\n\nexport interface StreamPublisher {\n connect(): Promise<void>;\n publish(topic: string, key: string, value: string): Promise<void>;\n /** Resolves once every accepted record has been acknowledged by the broker. */\n flush(): Promise<void>;\n disconnect(): Promise<void>;\n readonly publishedCount: number;\n readonly failedCount: number;\n}\n\nexport interface StreamSubscriber {\n connect(): Promise<void>;\n /**\n * Subscribes to `topics` and starts consuming. The handler is awaited, so a\n * throw leaves the offset uncommitted and the record is redelivered: every\n * handler must therefore be idempotent (the task service already is).\n */\n subscribe(\n topics: readonly string[],\n handler: (message: StreamMessage) => Promise<void>,\n ): Promise<void>;\n disconnect(): Promise<void>;\n readonly handledCount: number;\n readonly skippedCount: number;\n}\n\nfunction saslOptions(config: StreamConfig): SASLOptions | undefined {\n if (!config.sasl) return undefined;\n return {\n mechanism: config.sasl.mechanism,\n username: config.sasl.username,\n password: config.sasl.password,\n };\n}\n\nfunction createKafka(config: StreamConfig): Kafka {\n return new Kafka({\n clientId: config.clientId,\n brokers: [...config.brokers],\n ssl: config.ssl || undefined,\n sasl: saslOptions(config),\n connectionTimeout: config.connectionTimeoutMs,\n retry: { retries: config.retryCount, initialRetryTime: 300 },\n logLevel: logLevel.ERROR,\n });\n}\n\n/**\n * Creates the stream topics if the broker does not have them yet.\n *\n * Subscribing to a missing topic would make metadata refresh race the broker's\n * auto-creation, which surfaces as a transient `REPLICA_NOT_AVAILABLE` and\n * aborts startup. Creating the topics first, and waiting for their leaders, makes\n * the first subscribe deterministic on a fresh broker. Existing topics are left\n * untouched, including their partition count.\n *\n * Existing topics are filtered out before the create call: asking the broker to\n * create a topic that already exists is answered with a per-topic error, which\n * kafkajs reports at ERROR level. That noise on every restart trains operators\n * to ignore the stream logs, so the check is done here instead.\n */\nexport async function ensureTopics(\n config: StreamConfig,\n topics: readonly string[],\n logger?: StreamLogger,\n): Promise<void> {\n const admin = createKafka(config).admin();\n try {\n await admin.connect();\n } catch (error) {\n await admin.disconnect().catch(() => {});\n throw unavailableError(config, error);\n }\n\n try {\n const existing = new Set(await admin.listTopics());\n const missing = topics.filter((topic) => !existing.has(topic));\n if (missing.length === 0) return;\n\n const partitions = config.topicPartitions ?? 3;\n const replicationFactor = config.replicationFactor ?? 1;\n await admin.createTopics({\n topics: missing.map((topic) => ({\n topic,\n numPartitions: partitions,\n replicationFactor,\n })),\n waitForLeaders: true,\n });\n logger?.info(\n `Created stream topics: ${missing.join(', ')} ` +\n `(${partitions} partition(s), replication ${replicationFactor}).`,\n );\n } catch (error) {\n throw new StreamUnavailableError(\n `Could not create the stream topics (${topics.join(', ')}): ${(error as Error).message}`,\n config.brokers,\n error,\n );\n } finally {\n await admin.disconnect().catch(() => {});\n }\n}\n\n/**\n * Ordered, at-least-once producer.\n *\n * Sends are chained rather than fired in parallel so the broker sees one\n * agent's frames in the order they were produced. `acks: -1` means the promise\n * resolves only after every in-sync replica has the record, which is what makes\n * `flush()` a real guarantee before the process exits.\n */\nexport class KafkaPublisher implements StreamPublisher {\n private producer: Producer | null = null;\n private tail: Promise<void> = Promise.resolve();\n private sent = 0;\n private failed = 0;\n\n constructor(\n private readonly config: StreamConfig,\n private readonly logger?: StreamLogger,\n ) {}\n\n public get publishedCount(): number {\n return this.sent;\n }\n\n public get failedCount(): number {\n return this.failed;\n }\n\n public async connect(): Promise<void> {\n if (this.producer) return;\n const kafka = createKafka(this.config);\n try {\n const producer = kafka.producer({ allowAutoTopicCreation: true, idempotent: true });\n await producer.connect();\n this.producer = producer;\n } catch (error) {\n throw unavailableError(this.config, error);\n }\n this.logger?.debug(`Kafka producer connected to ${this.config.brokers.join(', ')}.`);\n }\n\n public publish(topic: string, key: string, value: string): Promise<void> {\n const producer = this.producer;\n if (!producer) {\n return Promise.reject(\n new StreamUnavailableError(\n 'The Kafka producer is not connected.',\n this.config.brokers,\n ),\n );\n }\n\n const send = this.tail.then(() =>\n producer.send({ topic, acks: -1, messages: [{ key, value }] }),\n );\n const settled = send.then(\n () => {\n this.sent += 1;\n },\n (error: unknown) => {\n this.failed += 1;\n this.logger?.error(\n `Could not publish to ${topic}: ${(error as Error).message}`,\n );\n throw error;\n },\n );\n // The chain must survive a failed send, or one broker hiccup would stall\n // every later record.\n this.tail = settled.catch(() => {});\n return settled;\n }\n\n public async flush(): Promise<void> {\n await this.tail;\n }\n\n public async disconnect(): Promise<void> {\n const producer = this.producer;\n this.producer = null;\n if (!producer) return;\n await this.tail.catch(() => {});\n await producer.disconnect().catch(() => {});\n }\n}\n\n/**\n * Consumer for one group.\n *\n * A poison record (bad envelope) is logged and skipped; anything else throws, so\n * the offset stays put and the record is retried. That is the only split where\n * at-least-once is safe: malformed input can never become authority, and valid\n * input is idempotent downstream.\n */\nexport class KafkaSubscriber implements StreamSubscriber {\n private consumer: Consumer | null = null;\n private started = false;\n private handled = 0;\n private skipped = 0;\n\n constructor(\n private readonly config: StreamConfig,\n private readonly groupId: string,\n private readonly options?: {\n readonly fromBeginning?: boolean;\n readonly logger?: StreamLogger;\n },\n ) {}\n\n public get handledCount(): number {\n return this.handled;\n }\n\n public get skippedCount(): number {\n return this.skipped;\n }\n\n public async connect(): Promise<void> {\n if (this.consumer) return;\n const kafka = createKafka(this.config);\n const consumer = kafka.consumer({\n groupId: this.groupId,\n sessionTimeout: 30_000,\n heartbeatInterval: 3_000,\n });\n try {\n await consumer.connect();\n } catch (error) {\n await consumer.disconnect().catch(() => {});\n throw unavailableError(this.config, error);\n }\n this.consumer = consumer;\n this.options?.logger?.debug(\n `Kafka consumer ${this.groupId} connected to ${this.config.brokers.join(', ')}.`,\n );\n }\n\n public async subscribe(\n topics: readonly string[],\n handler: (message: StreamMessage) => Promise<void>,\n ): Promise<void> {\n const consumer = this.consumer;\n if (!consumer) {\n throw new StreamUnavailableError(\n `Kafka consumer ${this.groupId} is not connected.`,\n this.config.brokers,\n );\n }\n if (this.started) {\n throw new Error(`Kafka consumer ${this.groupId} is already running.`);\n }\n\n const fromBeginning = this.options?.fromBeginning ?? false;\n for (const topic of topics) {\n await consumer.subscribe({ topic, fromBeginning });\n }\n\n await consumer.run({\n autoCommit: true,\n partitionsConsumedConcurrently: 1,\n eachMessage: async ({ topic, message }) => {\n if (!message.value) return;\n try {\n await handler({\n key: message.key ? message.key.toString() : null,\n value: message.value.toString(),\n });\n this.handled += 1;\n } catch (error) {\n if (error instanceof StreamProtocolError) {\n this.skipped += 1;\n this.options?.logger?.warn(\n `Skipping a malformed record on ${topic}: ${error.message}`,\n );\n return;\n }\n throw error;\n }\n },\n });\n this.started = true;\n }\n\n public async disconnect(): Promise<void> {\n const consumer = this.consumer;\n this.consumer = null;\n this.started = false;\n if (!consumer) return;\n await consumer.disconnect().catch(() => {});\n }\n}\n","/**\n * Topic names and partition keys.\n *\n * One topic per direction rather than one per agent: a topic per agent is\n * unbounded topic creation on a broker that is supposed to be boring. Ordering\n * is preserved per key instead, which is all the protocol needs:\n *\n * - commands are keyed by agent, so one agent's offers arrive in order;\n * - reports are keyed by task, so one task's pages, ack, progress and completion\n * arrive in order even when several agents report concurrently;\n * - results are keyed by task, for the same reason.\n */\nexport const STREAM_TOPICS = {\n /** Server to agent: offers, control frames, lease renewals, nacks. */\n commands: 'luxray.agent.commands',\n /**\n * Agent to server: page observations and acks/progress/checkpoints/completion.\n *\n * Pages share this topic with commands on purpose. Keyed by task, they land in\n * one partition, so the control plane applies every page before the report\n * that closes the same attempt. A `report` record always follows this order.\n */\n reports: 'luxray.agent.reports',\n /** Server to subscribers: durable lifecycle events, mirrored for consumers. */\n events: 'luxray.tasks.events',\n /** Server to subscribers: page observations, mirrored after they persist. */\n results: 'luxray.crawl.results',\n} as const;\n\nexport type StreamTopic = (typeof STREAM_TOPICS)[keyof typeof STREAM_TOPICS];\n\n/**\n * Every stream topic, in the order a fresh broker should be given them.\n *\n * Consumers create topics explicitly before subscribing: subscribing to a topic\n * that does not exist yet makes metadata refresh race topic creation, and a\n * control plane must not fail to start on that transient.\n */\nexport const STREAM_TOPIC_LIST: readonly StreamTopic[] = [\n STREAM_TOPICS.commands,\n STREAM_TOPICS.reports,\n STREAM_TOPICS.results,\n STREAM_TOPICS.events,\n];\n\n/** Each agent is its own consumer group, so it sees only its own commands. */\nexport function agentConsumerGroup(agentId: string): string {\n return `luxray-agent-${agentId}`;\n}\n\n/** All API instances share a group: a report is applied exactly once. */\nexport const API_REPORTS_GROUP = 'luxray-api-reports';\nexport const API_RESULTS_GROUP = 'luxray-api-results';\n","import type {\n AgentTaskCancelMessage,\n AgentTaskClientMessage,\n AgentTaskNackMessage,\n AgentTaskOfferMessage,\n AgentTaskPauseMessage,\n AgentTaskRenewedMessage,\n AgentTaskServerMessage,\n CrawlPageResult,\n StreamEnvelope,\n} from '@luxray/shared';\nimport {\n KafkaPublisher,\n KafkaSubscriber,\n STREAM_TOPICS,\n agentConsumerGroup,\n decodeEnvelope,\n encodeEnvelope,\n ensureTopics,\n type StreamConfig,\n type StreamMessage,\n type StreamPublisher,\n type StreamSubscriber,\n} from '@luxray/streaming';\n\nexport interface TaskStreamLogger {\n info: (...args: unknown[]) => void;\n warn: (...args: unknown[]) => void;\n error: (...args: unknown[]) => void;\n debug: (...args: unknown[]) => void;\n}\n\n/**\n * Receives the commands addressed to this agent.\n *\n * Handlers are synchronous by design: the executor's decisions (ack, run, stop)\n * are decisions about local state, and a Kafka handler that awaits a long job\n * would stall the partition behind it.\n */\nexport interface TaskStreamHandlers {\n readonly onOffer: (message: AgentTaskOfferMessage) => void;\n readonly onCancel: (message: AgentTaskCancelMessage) => void;\n readonly onPause: (message: AgentTaskPauseMessage) => void;\n readonly onRenewed: (message: AgentTaskRenewedMessage) => void;\n readonly onNack: (message: AgentTaskNackMessage) => void;\n}\n\nexport interface TaskStreamOptions {\n readonly config: StreamConfig;\n readonly agentId: string;\n readonly workspaceId: string;\n readonly handlers: TaskStreamHandlers;\n readonly logger?: TaskStreamLogger;\n /** Injected in tests so the stream can be exercised without a broker. */\n readonly publisher?: StreamPublisher;\n readonly subscriber?: StreamSubscriber;\n /** Injected in tests; the default creates the topics on the real broker. */\n readonly ensureTopics?: (config: StreamConfig) => Promise<void>;\n}\n\n/**\n * The agent's side of the Kafka transport.\n *\n * Reports and page results leave through the publisher; commands arrive on a\n * consumer group private to this agent. The group is what makes an offer\n * durable: a command produced while the agent is offline is still there when it\n * reconnects, whereas the old socket could only replay what the control plane\n * remembered to resend.\n */\nexport class TaskStream {\n private readonly publisher: StreamPublisher;\n private readonly subscriber: StreamSubscriber;\n private readonly handlers: TaskStreamHandlers;\n private readonly logger: TaskStreamLogger;\n private readonly prepareTopics: (config: StreamConfig) => Promise<void>;\n private connected = false;\n\n constructor(private readonly options: TaskStreamOptions) {\n this.publisher =\n options.publisher ?? new KafkaPublisher(options.config, options.logger);\n this.subscriber =\n options.subscriber ??\n new KafkaSubscriber(options.config, agentConsumerGroup(options.agentId), {\n fromBeginning: true,\n logger: options.logger,\n });\n this.handlers = options.handlers;\n this.prepareTopics =\n options.ensureTopics ??\n ((config) =>\n ensureTopics(config, [\n STREAM_TOPICS.commands,\n STREAM_TOPICS.reports,\n STREAM_TOPICS.results,\n ]));\n this.logger = options.logger ?? {\n info: () => {},\n warn: (...args: unknown[]) => console.warn(...args),\n error: (...args: unknown[]) => console.error(...args),\n debug: () => {},\n };\n }\n\n public get isConnected(): boolean {\n return this.connected;\n }\n\n public get publishedCount(): number {\n return this.publisher.publishedCount;\n }\n\n /**\n * Connects both sides before anything is produced.\n *\n * The subscriber connects first in spirit as well as in order: a command\n * published between \"agent is online\" and \"agent is consuming\" would still be\n * delivered later, but a producer that is not connected silently rejects\n * every report it is handed.\n */\n public async connect(): Promise<void> {\n if (this.connected) return;\n await this.subscriber.connect();\n try {\n await this.publisher.connect();\n // A fresh broker has no topics; creating them before subscribing avoids\n // the metadata race that would otherwise look like an unavailable broker.\n await this.prepareTopics(this.options.config);\n await this.subscriber.subscribe([STREAM_TOPICS.commands], (message) =>\n this.receive(message),\n );\n } catch (error) {\n await this.subscriber.disconnect().catch(() => {});\n await this.publisher.disconnect().catch(() => {});\n throw error;\n }\n this.connected = true;\n }\n\n /**\n * Publishes one execution report.\n *\n * Returns false when the stream is not connected, which the executor already\n * treats as a transient gap. A publish failure after acceptance is logged,\n * not thrown: the lease will expire and the control plane will reclaim the\n * task, which is the same outcome as a socket write that never arrived.\n */\n public report(message: AgentTaskClientMessage): boolean {\n if (!this.connected) return false;\n\n const envelope: StreamEnvelope<AgentTaskClientMessage> = {\n envelope: 1,\n kind: 'report',\n agentId: this.options.agentId,\n workspaceId: this.options.workspaceId,\n sentAt: Date.now(),\n payload: message,\n };\n\n // Keyed by task: one task's ack/progress/complete stay in order even when\n // several jobs on this agent publish at once.\n void this.publisher\n .publish(STREAM_TOPICS.reports, message.taskId, encodeEnvelope(envelope))\n .catch((error: unknown) => {\n this.logger.error(\n `Could not publish ${message.type} for task ${message.taskId}: ${(error as Error).message}`,\n );\n });\n return true;\n }\n\n /**\n * Streams one observed page while the crawl is still running.\n *\n * Pages share the topic with reports and are keyed by task, so the completion\n * that follows them lands in the same partition and can never overtake them.\n */\n public publishPage(page: CrawlPageResult): boolean {\n if (!this.connected) return false;\n\n const envelope: StreamEnvelope<CrawlPageResult> = {\n envelope: 1,\n kind: 'result',\n agentId: this.options.agentId,\n workspaceId: this.options.workspaceId,\n sentAt: Date.now(),\n payload: page,\n };\n\n void this.publisher\n .publish(STREAM_TOPICS.reports, page.taskId, encodeEnvelope(envelope))\n .catch((error: unknown) => {\n this.logger.warn(\n `Could not publish page ${page.sequence} of task ${page.taskId}: ${(error as Error).message}`,\n );\n });\n return true;\n }\n\n /** Waits until every accepted record was acknowledged by the broker. */\n public async flush(): Promise<void> {\n await this.publisher.flush();\n }\n\n public async close(): Promise<void> {\n if (!this.connected) {\n await this.publisher.disconnect().catch(() => {});\n await this.subscriber.disconnect().catch(() => {});\n return;\n }\n this.connected = false;\n // Flush before disconnecting, or a completion queued during shutdown is\n // dropped and the task sits RUNNING until its lease expires.\n await this.flush().catch((error: unknown) => {\n this.logger.error(`Could not flush the task stream: ${(error as Error).message}`);\n });\n await this.subscriber.disconnect();\n await this.publisher.disconnect();\n }\n\n private async receive(message: StreamMessage): Promise<void> {\n const envelope = decodeEnvelope<AgentTaskServerMessage>(message.value, 'task');\n\n // The group is private to this agent, so a foreign envelope means a\n // misconfigured producer, not a message for us. Ignoring it is safer than\n // executing a task this agent was never assigned.\n if (envelope.agentId !== this.options.agentId) {\n this.logger.warn(\n `Ignoring a ${envelope.payload.type} record addressed to agent ${envelope.agentId}.`,\n );\n return;\n }\n\n const payload = envelope.payload;\n switch (payload.type) {\n case 'task.offer':\n this.handlers.onOffer(payload);\n return;\n case 'task.cancel':\n this.handlers.onCancel(payload);\n return;\n case 'task.pause':\n this.handlers.onPause(payload);\n return;\n case 'task.renewed':\n this.handlers.onRenewed(payload);\n return;\n case 'task.nack':\n this.handlers.onNack(payload);\n return;\n default: {\n // A newer server may send a command this build does not know. Retrying\n // it would block every later command on this agent's partition, and it\n // can never succeed, so it is logged and dropped. The durable task row\n // is unaffected; the lease sweep is the backstop.\n const unknown = payload as { type?: unknown };\n this.logger.warn(`Ignoring an unknown task command: ${String(unknown.type)}.`);\n return;\n }\n }\n }\n}\n"],"mappings":";AAAA,SAAS,YAAY,cAAc,mBAAmB;AACtD,SAAS,YAAY;;;ACAf,IAAO,yBAAP,cAAsC,MAAK;EAK7B;EACS;EALF,OAAO;EAEhC,YACE,SACgB,SACS,OAAe;AAExC,UAAM,OAAO;AAHG,SAAA,UAAA;AACS,SAAA,QAAA;EAG3B;;AAII,IAAO,sBAAP,cAAmC,MAAK;EACnB,OAAO;;;;AD6I5B,SAAU,iBAAiB,QAAsB,OAAc;AACnE,QAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,SAAO,IAAI,uBACT,6BAA6B,OAAO,QAAQ,KAAK,IAAI,CAAC,KAAK,MAAM,wFAEjE,OAAO,SACP,KAAK;AAET;;;AEjKA,IAAM,QAA+B,CAAC,QAAQ,UAAU,SAAS,QAAQ;AAEnE,SAAU,eAAkB,UAA2B;AAC3D,SAAO,KAAK,UAAU,QAAQ;AAChC;AAaM,SAAU,eACd,KACA,UAA6C;AAE7C,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;EACzB,SAAS,OAAO;AACd,UAAM,IAAI,oBAAoB,uBAAwB,MAAgB,OAAO,EAAE;EACjF;AAEA,MAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GAAG;AAC1E,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,QAAM,SAAS;AACf,MAAI,OAAO,UAAU,MAAM,GAAG;AAC5B,UAAM,IAAI,oBAAoB,iCAAiC,OAAO,OAAO,UAAU,CAAC,CAAC,GAAG;EAC9F;AAEA,QAAM,OAAO,OAAO,MAAM;AAC1B,MAAI,OAAO,SAAS,YAAY,CAAC,MAAM,SAAS,IAAkB,GAAG;AACnE,UAAM,IAAI,oBAAoB,wBAAwB,OAAO,IAAI,CAAC,GAAG;EACvE;AACA,MAAI,aAAa,QAAW;AAC1B,UAAM,UAAU,OAAO,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC5D,QAAI,CAAC,QAAQ,SAAS,IAAkB,GAAG;AACzC,YAAM,IAAI,oBACR,cAAc,QAAQ,KAAK,MAAM,CAAC,gBAAgB,IAAI,GAAG;IAE7D;EACF;AAEA,QAAM,UAAU,OAAO,SAAS;AAGhC,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AACA,OAAK,SAAS,UAAU,SAAS,aAAa,QAAQ,WAAW,GAAG;AAClE,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,QAAM,cAAc,OAAO,aAAa;AACxC,MAAI,OAAO,gBAAgB,YAAY,YAAY,WAAW,GAAG;AAC/D,UAAM,IAAI,oBAAoB,8BAA8B;EAC9D;AAEA,QAAM,SAAS,OAAO,QAAQ;AAC9B,MAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,MAAM,GAAG;AAC1D,UAAM,IAAI,oBAAoB,yBAAyB;EACzD;AAEA,QAAM,UAAU,OAAO,SAAS;AAChC,MAAI,YAAY,QAAQ,OAAO,YAAY,UAAU;AACnD,UAAM,IAAI,oBAAoB,0BAA0B;EAC1D;AAEA,SAAO;IACL,UAAU;IACV;IACA;IACA;IACA;IACA;;AAEJ;;;ACtFA,SAAS,OAAO,gBAAgE;AA2ChF,SAAS,YAAY,QAAoB;AACvC,MAAI,CAAC,OAAO;AAAM,WAAO;AACzB,SAAO;IACL,WAAW,OAAO,KAAK;IACvB,UAAU,OAAO,KAAK;IACtB,UAAU,OAAO,KAAK;;AAE1B;AAEA,SAAS,YAAY,QAAoB;AACvC,SAAO,IAAI,MAAM;IACf,UAAU,OAAO;IACjB,SAAS,CAAC,GAAG,OAAO,OAAO;IAC3B,KAAK,OAAO,OAAO;IACnB,MAAM,YAAY,MAAM;IACxB,mBAAmB,OAAO;IAC1B,OAAO,EAAE,SAAS,OAAO,YAAY,kBAAkB,IAAG;IAC1D,UAAU,SAAS;GACpB;AACH;AAgBA,eAAsB,aACpB,QACA,QACA,QAAqB;AAErB,QAAM,QAAQ,YAAY,MAAM,EAAE,MAAK;AACvC,MAAI;AACF,UAAM,MAAM,QAAO;EACrB,SAAS,OAAO;AACd,UAAM,MAAM,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;AACvC,UAAM,iBAAiB,QAAQ,KAAK;EACtC;AAEA,MAAI;AACF,UAAM,WAAW,IAAI,IAAI,MAAM,MAAM,WAAU,CAAE;AACjD,UAAM,UAAU,OAAO,OAAO,CAAC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC;AAC7D,QAAI,QAAQ,WAAW;AAAG;AAE1B,UAAM,aAAa,OAAO,mBAAmB;AAC7C,UAAM,oBAAoB,OAAO,qBAAqB;AACtD,UAAM,MAAM,aAAa;MACvB,QAAQ,QAAQ,IAAI,CAAC,WAAW;QAC9B;QACA,eAAe;QACf;QACA;MACF,gBAAgB;KACjB;AACD,YAAQ,KACN,0BAA0B,QAAQ,KAAK,IAAI,CAAC,KACtC,UAAU,8BAA8B,iBAAiB,IAAI;EAEvE,SAAS,OAAO;AACd,UAAM,IAAI,uBACR,uCAAuC,OAAO,KAAK,IAAI,CAAC,MAAO,MAAgB,OAAO,IACtF,OAAO,SACP,KAAK;EAET;AACE,UAAM,MAAM,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EACzC;AACF;AAUM,IAAO,iBAAP,MAAqB;EAON;EACA;EAPX,WAA4B;EAC5B,OAAsB,QAAQ,QAAO;EACrC,OAAO;EACP,SAAS;EAEjB,YACmB,QACA,QAAqB;AADrB,SAAA,SAAA;AACA,SAAA,SAAA;EAChB;EAEH,IAAW,iBAAc;AACvB,WAAO,KAAK;EACd;EAEA,IAAW,cAAW;AACpB,WAAO,KAAK;EACd;EAEO,MAAM,UAAO;AAClB,QAAI,KAAK;AAAU;AACnB,UAAM,QAAQ,YAAY,KAAK,MAAM;AACrC,QAAI;AACF,YAAM,WAAW,MAAM,SAAS,EAAE,wBAAwB,MAAM,YAAY,KAAI,CAAE;AAClF,YAAM,SAAS,QAAO;AACtB,WAAK,WAAW;IAClB,SAAS,OAAO;AACd,YAAM,iBAAiB,KAAK,QAAQ,KAAK;IAC3C;AACA,SAAK,QAAQ,MAAM,+BAA+B,KAAK,OAAO,QAAQ,KAAK,IAAI,CAAC,GAAG;EACrF;EAEO,QAAQ,OAAe,KAAa,OAAa;AACtD,UAAM,WAAW,KAAK;AACtB,QAAI,CAAC,UAAU;AACb,aAAO,QAAQ,OACb,IAAI,uBACF,wCACA,KAAK,OAAO,OAAO,CACpB;IAEL;AAEA,UAAM,OAAO,KAAK,KAAK,KAAK,MAC1B,SAAS,KAAK,EAAE,OAAO,MAAM,IAAI,UAAU,CAAC,EAAE,KAAK,MAAK,CAAE,EAAC,CAAE,CAAC;AAEhE,UAAM,UAAU,KAAK,KACnB,MAAK;AACH,WAAK,QAAQ;IACf,GACA,CAAC,UAAkB;AACjB,WAAK,UAAU;AACf,WAAK,QAAQ,MACX,wBAAwB,KAAK,KAAM,MAAgB,OAAO,EAAE;AAE9D,YAAM;IACR,CAAC;AAIH,SAAK,OAAO,QAAQ,MAAM,MAAK;IAAE,CAAC;AAClC,WAAO;EACT;EAEO,MAAM,QAAK;AAChB,UAAM,KAAK;EACb;EAEO,MAAM,aAAU;AACrB,UAAM,WAAW,KAAK;AACtB,SAAK,WAAW;AAChB,QAAI,CAAC;AAAU;AACf,UAAM,KAAK,KAAK,MAAM,MAAK;IAAE,CAAC;AAC9B,UAAM,SAAS,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EAC5C;;AAWI,IAAO,kBAAP,MAAsB;EAOP;EACA;EACA;EARX,WAA4B;EAC5B,UAAU;EACV,UAAU;EACV,UAAU;EAElB,YACmB,QACA,SACA,SAGhB;AALgB,SAAA,SAAA;AACA,SAAA,UAAA;AACA,SAAA,UAAA;EAIhB;EAEH,IAAW,eAAY;AACrB,WAAO,KAAK;EACd;EAEA,IAAW,eAAY;AACrB,WAAO,KAAK;EACd;EAEO,MAAM,UAAO;AAClB,QAAI,KAAK;AAAU;AACnB,UAAM,QAAQ,YAAY,KAAK,MAAM;AACrC,UAAM,WAAW,MAAM,SAAS;MAC9B,SAAS,KAAK;MACd,gBAAgB;MAChB,mBAAmB;KACpB;AACD,QAAI;AACF,YAAM,SAAS,QAAO;IACxB,SAAS,OAAO;AACd,YAAM,SAAS,WAAU,EAAG,MAAM,MAAK;MAAE,CAAC;AAC1C,YAAM,iBAAiB,KAAK,QAAQ,KAAK;IAC3C;AACA,SAAK,WAAW;AAChB,SAAK,SAAS,QAAQ,MACpB,kBAAkB,KAAK,OAAO,iBAAiB,KAAK,OAAO,QAAQ,KAAK,IAAI,CAAC,GAAG;EAEpF;EAEO,MAAM,UACX,QACA,SAAkD;AAElD,UAAM,WAAW,KAAK;AACtB,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,uBACR,kBAAkB,KAAK,OAAO,sBAC9B,KAAK,OAAO,OAAO;IAEvB;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB,KAAK,OAAO,sBAAsB;IACtE;AAEA,UAAM,gBAAgB,KAAK,SAAS,iBAAiB;AACrD,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,UAAU,EAAE,OAAO,cAAa,CAAE;IACnD;AAEA,UAAM,SAAS,IAAI;MACjB,YAAY;MACZ,gCAAgC;MAChC,aAAa,OAAO,EAAE,OAAO,QAAO,MAAM;AACxC,YAAI,CAAC,QAAQ;AAAO;AACpB,YAAI;AACF,gBAAM,QAAQ;YACZ,KAAK,QAAQ,MAAM,QAAQ,IAAI,SAAQ,IAAK;YAC5C,OAAO,QAAQ,MAAM,SAAQ;WAC9B;AACD,eAAK,WAAW;QAClB,SAAS,OAAO;AACd,cAAI,iBAAiB,qBAAqB;AACxC,iBAAK,WAAW;AAChB,iBAAK,SAAS,QAAQ,KACpB,kCAAkC,KAAK,KAAK,MAAM,OAAO,EAAE;AAE7D;UACF;AACA,gBAAM;QACR;MACF;KACD;AACD,SAAK,UAAU;EACjB;EAEO,MAAM,aAAU;AACrB,UAAM,WAAW,KAAK;AACtB,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,QAAI,CAAC;AAAU;AACf,UAAM,SAAS,WAAU,EAAG,MAAM,MAAK;IAAE,CAAC;EAC5C;;;;ACzSK,IAAM,gBAAgB;;EAE3B,UAAU;;;;;;;;EAQV,SAAS;;EAET,QAAQ;;EAER,SAAS;;AAYJ,IAAM,oBAA4C;EACvD,cAAc;EACd,cAAc;EACd,cAAc;EACd,cAAc;;AAIV,SAAU,mBAAmB,SAAe;AAChD,SAAO,gBAAgB,OAAO;AAChC;;;ACqBO,IAAM,aAAN,MAAiB;AAAA,EAQtB,YAA6B,SAA4B;AAA5B;AAC3B,SAAK,YACH,QAAQ,aAAa,IAAI,eAAe,QAAQ,QAAQ,QAAQ,MAAM;AACxE,SAAK,aACH,QAAQ,cACR,IAAI,gBAAgB,QAAQ,QAAQ,mBAAmB,QAAQ,OAAO,GAAG;AAAA,MACvE,eAAe;AAAA,MACf,QAAQ,QAAQ;AAAA,IAClB,CAAC;AACH,SAAK,WAAW,QAAQ;AACxB,SAAK,gBACH,QAAQ,iBACP,CAAC,WACA,aAAa,QAAQ;AAAA,MACnB,cAAc;AAAA,MACd,cAAc;AAAA,MACd,cAAc;AAAA,IAChB,CAAC;AACL,SAAK,SAAS,QAAQ,UAAU;AAAA,MAC9B,MAAM,MAAM;AAAA,MAAC;AAAA,MACb,MAAM,IAAI,SAAoB,QAAQ,KAAK,GAAG,IAAI;AAAA,MAClD,OAAO,IAAI,SAAoB,QAAQ,MAAM,GAAG,IAAI;AAAA,MACpD,OAAO,MAAM;AAAA,MAAC;AAAA,IAChB;AAAA,EACF;AAAA,EAxB6B;AAAA,EAPZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT,YAAY;AAAA,EA4BpB,IAAW,cAAuB;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAW,iBAAyB;AAClC,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAa,UAAyB;AACpC,QAAI,KAAK,UAAW;AACpB,UAAM,KAAK,WAAW,QAAQ;AAC9B,QAAI;AACF,YAAM,KAAK,UAAU,QAAQ;AAG7B,YAAM,KAAK,cAAc,KAAK,QAAQ,MAAM;AAC5C,YAAM,KAAK,WAAW;AAAA,QAAU,CAAC,cAAc,QAAQ;AAAA,QAAG,CAAC,YACzD,KAAK,QAAQ,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,YAAM,KAAK,WAAW,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACjD,YAAM,KAAK,UAAU,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAChD,YAAM;AAAA,IACR;AACA,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUO,OAAO,SAA0C;AACtD,QAAI,CAAC,KAAK,UAAW,QAAO;AAE5B,UAAM,WAAmD;AAAA,MACvD,UAAU;AAAA,MACV,MAAM;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,aAAa,KAAK,QAAQ;AAAA,MAC1B,QAAQ,KAAK,IAAI;AAAA,MACjB,SAAS;AAAA,IACX;AAIA,SAAK,KAAK,UACP,QAAQ,cAAc,SAAS,QAAQ,QAAQ,eAAe,QAAQ,CAAC,EACvE,MAAM,CAAC,UAAmB;AACzB,WAAK,OAAO;AAAA,QACV,qBAAqB,QAAQ,IAAI,aAAa,QAAQ,MAAM,KAAM,MAAgB,OAAO;AAAA,MAC3F;AAAA,IACF,CAAC;AACH,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,YAAY,MAAgC;AACjD,QAAI,CAAC,KAAK,UAAW,QAAO;AAE5B,UAAM,WAA4C;AAAA,MAChD,UAAU;AAAA,MACV,MAAM;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,aAAa,KAAK,QAAQ;AAAA,MAC1B,QAAQ,KAAK,IAAI;AAAA,MACjB,SAAS;AAAA,IACX;AAEA,SAAK,KAAK,UACP,QAAQ,cAAc,SAAS,KAAK,QAAQ,eAAe,QAAQ,CAAC,EACpE,MAAM,CAAC,UAAmB;AACzB,WAAK,OAAO;AAAA,QACV,0BAA0B,KAAK,QAAQ,YAAY,KAAK,MAAM,KAAM,MAAgB,OAAO;AAAA,MAC7F;AAAA,IACF,CAAC;AACH,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAa,QAAuB;AAClC,UAAM,KAAK,UAAU,MAAM;AAAA,EAC7B;AAAA,EAEA,MAAa,QAAuB;AAClC,QAAI,CAAC,KAAK,WAAW;AACnB,YAAM,KAAK,UAAU,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAChD,YAAM,KAAK,WAAW,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACjD;AAAA,IACF;AACA,SAAK,YAAY;AAGjB,UAAM,KAAK,MAAM,EAAE,MAAM,CAAC,UAAmB;AAC3C,WAAK,OAAO,MAAM,oCAAqC,MAAgB,OAAO,EAAE;AAAA,IAClF,CAAC;AACD,UAAM,KAAK,WAAW,WAAW;AACjC,UAAM,KAAK,UAAU,WAAW;AAAA,EAClC;AAAA,EAEA,MAAc,QAAQ,SAAuC;AAC3D,UAAM,WAAW,eAAuC,QAAQ,OAAO,MAAM;AAK7E,QAAI,SAAS,YAAY,KAAK,QAAQ,SAAS;AAC7C,WAAK,OAAO;AAAA,QACV,cAAc,SAAS,QAAQ,IAAI,8BAA8B,SAAS,OAAO;AAAA,MACnF;AACA;AAAA,IACF;AAEA,UAAM,UAAU,SAAS;AACzB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,aAAK,SAAS,QAAQ,OAAO;AAC7B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,SAAS,OAAO;AAC9B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,QAAQ,OAAO;AAC7B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,UAAU,OAAO;AAC/B;AAAA,MACF,KAAK;AACH,aAAK,SAAS,OAAO,OAAO;AAC5B;AAAA,MACF,SAAS;AAKP,cAAM,UAAU;AAChB,aAAK,OAAO,KAAK,qCAAqC,OAAO,QAAQ,IAAI,CAAC,GAAG;AAC7E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":[]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "luxray-agent",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Official local execution agent CLI for Luxray autonomous API intelligence & exploration",
5
5
  "type": "module",
6
6
  "bin": {