@opensteer/runtime-core 0.1.7 → 0.2.1

package/README.md

@@ -0,0 +1,13 @@
+# @opensteer/runtime-core
+
+Shared semantic runtime for Opensteer local and cloud execution.
+
+This package is primarily for engine authors and advanced integrations that need
+the lower-level runtime orchestration used by `opensteer`.
+
+```bash
+pnpm add @opensteer/runtime-core
+```
+
+Most users should install [`opensteer`](../opensteer/README.md), which bundles the
+public SDK and CLI on top of this runtime layer.

package/dist/index.cjs

@@ -41,7 +41,7 @@ var vm__default = /*#__PURE__*/_interopDefault(vm);
 
 // package.json
 var package_default = {
-  version: "0.1.7"};
+  version: "0.2.1"};
 
 // src/version.ts
 var OPENSTEER_RUNTIME_CORE_VERSION = package_default.version;
@@ -2386,6 +2386,16 @@ var opensteerNetworkQueryOutputSchema = objectSchema(
     required: ["records"]
   }
 );
+var opensteerNetworkDetailInputSchema = objectSchema(
+  {
+    recordId: stringSchema({ minLength: 1 }),
+    probe: { type: "boolean" }
+  },
+  {
+    title: "OpensteerNetworkDetailInput",
+    required: ["recordId"]
+  }
+);
 var opensteerParsedCookieSchema = objectSchema(
   {
     name: stringSchema({ minLength: 1 }),
@@ -2491,7 +2501,7 @@ objectSchema(
   }
 );
 var opensteerSessionFetchTransportSchema = enumSchema(
-  ["auto", "direct", "matched-tls", "page"],
+  ["auto", "direct", "matched-tls", "context", "page"],
   {
     title: "OpensteerSessionFetchTransport"
   }
@@ -6262,15 +6272,6 @@ var opensteerPageSnapshotOutputSchema = objectSchema(
     required: ["url", "title", "mode", "html", "counters"]
   }
 );
-var opensteerNetworkDetailInputSchema = objectSchema(
-  {
-    recordId: stringSchema({ minLength: 1 })
-  },
-  {
-    title: "OpensteerNetworkDetailInput",
-    required: ["recordId"]
-  }
-);
 var opensteerComputerMouseButtonSchema = enumSchema(
   ["left", "middle", "right"],
   {
@@ -6840,6 +6841,7 @@ var opensteerSemanticOperationSpecificationsBase = [
         case "direct":
           return [];
         case "matched-tls":
+        case "context":
           return ["inspect.cookies"];
         case "page":
           return ["pages.manage"];
@@ -7539,7 +7541,6 @@ var DEFAULT_TIMEOUTS = {
   "dom.extract": 15e3,
   "network.query": 15e3,
   "network.detail": 15e3,
-  "network.replay": 3e4,
   "scripts.capture": 15e3,
   "session.cookies": 1e4,
   "session.storage": 1e4,
@@ -8361,13 +8362,6 @@ var SqliteSavedNetworkStore = class {
   }
   async save(records, options) {
     const database = await this.requireDatabase();
-    const readExisting = database.prepare(`
-        SELECT record_id
-        FROM saved_network_records
-        WHERE session_ref = @session_ref
-          AND page_ref_key = @page_ref_key
-          AND request_id = @request_id
-      `);
     const upsertRecord = database.prepare(buildSavedNetworkUpsertSql(options.bodyWriteMode));
     const insertTag = database.prepare(`
       INSERT OR IGNORE INTO saved_network_tags (record_id, tag)
@@ -8378,14 +8372,8 @@ var SqliteSavedNetworkStore = class {
       for (const entry of records) {
         const url = new URL(entry.record.url);
         const pageRefKey = entry.record.pageRef ?? "";
-        const existing = readExisting.get({
-          session_ref: entry.record.sessionRef,
-          page_ref_key: pageRefKey,
-          request_id: entry.record.requestId
-        }) ?? void 0;
-        const recordId = existing?.record_id ?? entry.recordId;
         upsertRecord.run({
-          record_id: recordId,
+          record_id: entry.recordId,
           request_id: entry.record.requestId,
           session_ref: entry.record.sessionRef,
           page_ref: entry.record.pageRef ?? null,
@@ -8430,7 +8418,7 @@ var SqliteSavedNetworkStore = class {
         }
         for (const currentTag of tags) {
           const result = insertTag.run({
-            record_id: recordId,
+            record_id: entry.recordId,
             tag: currentTag
           });
           savedCount += result.changes ?? 0;
@@ -8533,6 +8521,49 @@ var SqliteSavedNetworkStore = class {
       return cleared;
     });
   }
+  async *iterateBatches(options = {}) {
+    const database = await this.requireDatabase();
+    const batchSize = Math.max(1, Math.min(options.batchSize ?? 500, 1e3));
+    let cursor;
+    while (true) {
+      const rows = database.prepare(
+        `
+      SELECT
+        r.*,
+        GROUP_CONCAT(t.tag, '${TAG_DELIMITER}') AS tags
+      FROM saved_network_records r
+      LEFT JOIN saved_network_tags t
+        ON t.record_id = r.record_id
+      ${cursor === void 0 ? "" : "WHERE r.saved_at > ? OR (r.saved_at = ? AND r.record_id > ?)"}
+      GROUP BY r.record_id
+      ORDER BY r.saved_at ASC, r.record_id ASC
+      LIMIT ?
+    `
+      ).all(
+        ...cursor === void 0 ? [] : [cursor.savedAt, cursor.savedAt, cursor.recordId],
+        batchSize
+      );
+      if (rows.length === 0) {
+        return;
+      }
+      yield rows.map((row) => inflateSavedNetworkRow(row, options.includeBodies ?? true));
+      const lastRow = rows.at(-1);
+      if (lastRow === void 0) {
+        return;
+      }
+      cursor = {
+        savedAt: lastRow.saved_at,
+        recordId: lastRow.record_id
+      };
+    }
+  }
+  close() {
+    if (this.database !== void 0) {
+      closeSqliteDatabase(this.database);
+      this.database = void 0;
+      this.databaseInitialization = void 0;
+    }
+  }
   async requireDatabase() {
     if (this.database) {
       return this.database;
@@ -8617,15 +8648,6 @@ var SqliteSavedNetworkStore = class {
         saved_at INTEGER NOT NULL
       );
 
-      CREATE UNIQUE INDEX IF NOT EXISTS saved_network_records_scope_request
-        ON saved_network_records (session_ref, page_ref_key, request_id);
-
-      CREATE INDEX IF NOT EXISTS saved_network_records_saved_at
-        ON saved_network_records (saved_at DESC);
-
-      CREATE INDEX IF NOT EXISTS saved_network_records_capture
-        ON saved_network_records (capture);
-
       CREATE TABLE IF NOT EXISTS saved_network_tags (
         record_id TEXT NOT NULL REFERENCES saved_network_records(record_id) ON DELETE CASCADE,
         tag TEXT NOT NULL,
@@ -8635,6 +8657,19 @@ var SqliteSavedNetworkStore = class {
       CREATE INDEX IF NOT EXISTS saved_network_tags_tag
         ON saved_network_tags (tag);
     `);
+    database.exec(`DROP INDEX IF EXISTS saved_network_records_scope_request`);
+    database.exec(`
+      CREATE INDEX IF NOT EXISTS saved_network_records_scope_request
+        ON saved_network_records (session_ref, page_ref_key, request_id)
+    `);
+    database.exec(`
+      CREATE INDEX IF NOT EXISTS saved_network_records_saved_at
+        ON saved_network_records (saved_at DESC)
+    `);
+    database.exec(`
+      CREATE INDEX IF NOT EXISTS saved_network_records_capture
+        ON saved_network_records (capture)
+    `);
     this.ensureColumn(
       database,
       "saved_network_records",
@@ -8954,6 +8989,14 @@ function withSqliteTransaction(database, task) {
 function createSavedNetworkStore(rootPath) {
   return new SqliteSavedNetworkStore(rootPath);
 }
+async function* iterateSavedNetworkRecordBatches(rootPath, options = {}) {
+  const store = new SqliteSavedNetworkStore(rootPath);
+  try {
+    yield* store.iterateBatches(options);
+  } finally {
+    store.close();
+  }
+}
 function normalizeContext(context) {
   return {
     ...context?.sessionRef === void 0 ? {} : { sessionRef: context.sessionRef },
@@ -17820,6 +17863,9 @@ var MUTATION_CAPTURE_FINALIZE_TIMEOUT_MS = 5e3;
 var PERSISTED_NETWORK_FLUSH_TIMEOUT_MS = 5e3;
 var PENDING_OPERATION_EVENT_CAPTURE_LIMIT = 64;
 var PENDING_OPERATION_EVENT_CAPTURE_SKEW_MS = 1e3;
+var REPLAY_PROBE_MIN_ATTEMPT_TIMEOUT_MS = 3e3;
+var REPLAY_PROBE_MAX_ATTEMPT_TIMEOUT_MS = 15e3;
+var REPLAY_PROBE_POST_SUCCESS_ATTEMPT_TIMEOUT_MS = 5e3;
 var OpensteerSessionRuntime = class {
   workspace;
   rootPath;
@@ -18681,26 +18727,27 @@ var OpensteerSessionRuntime = class {
     }
   }
   async queryNetwork(input = {}, options = {}) {
-    assertValidSemanticOperationInput("network.query", input);
+    const normalizedInput = normalizeNetworkQueryInput(input);
+    assertValidSemanticOperationInput("network.query", normalizedInput);
     const root = await this.ensureRoot();
     const startedAt = Date.now();
     try {
       const output = await this.runWithOperationTimeout(
         "network.query",
         async (timeout) => {
-          await this.syncPersistedNetworkSelection(timeout, input, {
+          await this.syncPersistedNetworkSelection(timeout, normalizedInput, {
             includeBodies: false
           });
           const rawRecords = await timeout.runStep(
             () => root.registry.savedNetwork.query({
-              ...this.toSavedNetworkQueryInput(input),
-              limit: Math.max(input.limit ?? 50, 1e3)
+              ...this.toSavedNetworkQueryInput(normalizedInput),
+              limit: Math.max(normalizedInput.limit ?? 50, 1e3)
             })
           );
-          const filtered = filterNetworkSummaryRecords(rawRecords, input);
+          const filtered = filterNetworkSummaryRecords(rawRecords, normalizedInput);
           const sorted = sortPersistedNetworkRecordsChronologically(filtered);
-          const sliced = sliceNetworkSummaryWindow(sorted, input);
-          const limited = sliced.slice(0, Math.max(1, Math.min(input.limit ?? 50, 200)));
+          const sliced = sliceNetworkSummaryWindow(sorted, normalizedInput);
+          const limited = sliced.slice(0, Math.max(1, Math.min(normalizedInput.limit ?? 50, 200)));
           const summaries = await this.buildNetworkSummaryRecords(limited, timeout);
           return {
             records: summaries
@@ -18714,9 +18761,9 @@ var OpensteerSessionRuntime = class {
         completedAt: Date.now(),
         outcome: "ok",
         data: {
-          limit: input.limit ?? 50,
-          ...input.capture === void 0 ? {} : { capture: input.capture },
-          ...input.json === true ? { json: true } : {},
+          limit: normalizedInput.limit ?? 50,
+          ...normalizedInput.capture === void 0 ? {} : { capture: normalizedInput.capture },
+          ...normalizedInput.json === true ? { json: true } : {},
           count: output.records.length
         },
         context: buildRuntimeTraceContext({
@@ -18741,12 +18788,13 @@ var OpensteerSessionRuntime = class {
     }
   }
   async getNetworkDetail(input, options = {}) {
+    const normalizedRecordId = normalizeNetworkRecordId(input.recordId);
     const startedAt = Date.now();
     try {
       const output = await this.runWithOperationTimeout(
         "network.detail",
         async (timeout) => {
-          const record = await this.resolveNetworkRecordByRecordId(input.recordId, timeout, {
+          const record = await this.resolveNetworkRecordByRecordId(normalizedRecordId, timeout, {
             includeBodies: true,
             redactSecretHeaders: false
           });
@@ -18765,8 +18813,8 @@ var OpensteerSessionRuntime = class {
         completedAt: Date.now(),
         outcome: "ok",
         data: {
-          recordId: input.recordId,
-          status: output.summary.status,
+          recordId: normalizedRecordId,
+          ...output.summary.status === void 0 ? {} : { status: output.summary.status },
           url: output.summary.url
         },
         context: buildRuntimeTraceContext({
@@ -20170,7 +20218,9 @@ var OpensteerSessionRuntime = class {
       ...graphql.persisted === void 0 ? {} : { persisted: graphql.persisted },
       ...graphqlVariables === void 0 ? {} : { variables: graphqlVariables }
     };
-    const requestBody = shouldShowRequestBody(record.record.method) && record.record.requestBody !== void 0 ? buildStructuredBodyPreview(record.record.requestBody, record.record.requestHeaders) : void 0;
+    const requestBody = shouldShowRequestBody(record.record.method) && record.record.requestBody !== void 0 ? buildStructuredBodyPreview(record.record.requestBody, record.record.requestHeaders, {
+      truncateData: false
+    }) : void 0;
     const responseBody = record.record.responseBody === void 0 ? void 0 : buildStructuredBodyPreview(record.record.responseBody, record.record.responseHeaders);
     const notes = detectNetworkRecordNotes(record);
     return {
@@ -20200,8 +20250,18 @@ var OpensteerSessionRuntime = class {
     let recommended;
     for (const transport of REPLAY_TRANSPORT_LADDER) {
       const attemptStartedAt = Date.now();
+      const attemptTimeoutMs = resolveReplayProbeAttemptTimeoutMs({
+        remainingMs: timeout.remainingMs(),
+        transportsRemaining: REPLAY_TRANSPORT_LADDER.length - attempts.length,
+        recommendedFound: recommended !== void 0
+      });
       try {
-        const output = await this.executeReplayTransportAttempt(transport, request, timeout);
+        const output = await this.executeReplayTransportAttemptWithinBudget(
+          transport,
+          request,
+          timeout,
+          attemptTimeoutMs
+        );
         const ok = matchesSuccessFingerprintFromProtocolResponse(output.response, fingerprint);
         attempts.push({
           transport,
@@ -20217,7 +20277,7 @@ var OpensteerSessionRuntime = class {
           transport,
           ok: false,
           durationMs: Date.now() - attemptStartedAt,
-          error: normalizeRuntimeErrorMessage(error)
+          error: normalizeProbeTransportAttemptError(transport, error, attemptTimeoutMs)
         });
       }
     }
@@ -20421,6 +20481,23 @@ var OpensteerSessionRuntime = class {
       }
     }
   }
+  async executeReplayTransportAttemptWithinBudget(transport, request, timeout, attemptTimeoutMs) {
+    if (attemptTimeoutMs === void 0) {
+      return this.executeReplayTransportAttempt(transport, request, timeout);
+    }
+    return runWithPolicyTimeout(
+      {
+        resolveTimeoutMs() {
+          return attemptTimeoutMs;
+        }
+      },
+      {
+        operation: timeout.operation,
+        signal: timeout.signal
+      },
+      (attemptTimeout) => this.executeReplayTransportAttempt(transport, request, attemptTimeout)
+    );
+  }
   async executeFetchTransportAttempt(transport, request, timeout, input) {
     let prepared = finalizeMaterializedTransportRequest(request, transport);
     if (input.cookies !== false && transport === "direct-http" && this.currentBinding() !== void 0) {
@@ -21335,10 +21412,15 @@ var OpensteerSessionRuntime = class {
     return this.observationSessionId ?? this.sessionRef;
   }
   runWithOperationTimeout(operation, callback, options = {}) {
+    const timeoutPolicy = options.timeoutMs === void 0 ? this.policy.timeout : {
+      resolveTimeoutMs() {
+        return options.timeoutMs;
+      }
+    };
     const existingCollector = this.operationEventStorage.getStore();
     if (existingCollector !== void 0) {
       return runWithPolicyTimeout(
-        this.policy.timeout,
+        timeoutPolicy,
         {
           operation,
           ...options.signal === void 0 ? {} : { signal: options.signal }
@@ -21351,7 +21433,7 @@ var OpensteerSessionRuntime = class {
     return this.operationEventStorage.run(collector, async () => {
       try {
         return await runWithPolicyTimeout(
-          this.policy.timeout,
+          timeoutPolicy,
           {
             operation,
             ...options.signal === void 0 ? {} : { signal: options.signal }
@@ -21521,6 +21603,21 @@ function buildEngineNetworkRecordFilters(input) {
 function normalizeNetworkStatusFilter(status) {
   return String(status);
 }
+function normalizeNetworkQueryInput(input) {
+  return {
+    ...input,
+    ...input.recordId === void 0 ? {} : { recordId: normalizeNetworkRecordId(input.recordId) },
+    ...input.before === void 0 ? {} : { before: normalizeNetworkRecordId(input.before) },
+    ...input.after === void 0 ? {} : { after: normalizeNetworkRecordId(input.after) }
+  };
+}
+function normalizeNetworkRecordId(recordId) {
+  const trimmed = recordId.trim();
+  if (trimmed.length === 0 || trimmed.startsWith("record:")) {
+    return trimmed;
+  }
+  return `record:${trimmed}`;
+}
 function resolveLiveQueryRequestIds(input, history) {
   const requestIdCandidates = [];
   if (input.recordId !== void 0) {
@@ -21733,6 +21830,20 @@ var REPLAY_TRANSPORT_LADDER = [
   "context-http",
   "page-http"
 ];
+function resolveReplayProbeAttemptTimeoutMs(input) {
+  const attemptCapMs = input.recommendedFound ? REPLAY_PROBE_POST_SUCCESS_ATTEMPT_TIMEOUT_MS : REPLAY_PROBE_MAX_ATTEMPT_TIMEOUT_MS;
+  const clampedRemaining = input.remainingMs === void 0 ? void 0 : Math.max(0, input.remainingMs);
+  if (clampedRemaining === 0) {
+    return 0;
+  }
+  if (clampedRemaining === void 0) {
+    return attemptCapMs;
+  }
+  const sliceMs = Math.floor(clampedRemaining / Math.max(1, input.transportsRemaining));
+  const minimumBudgetAffordable = clampedRemaining >= REPLAY_PROBE_MIN_ATTEMPT_TIMEOUT_MS * input.transportsRemaining;
+  const attemptBudgetMs = minimumBudgetAffordable ? Math.max(REPLAY_PROBE_MIN_ATTEMPT_TIMEOUT_MS, sliceMs) : sliceMs;
+  return Math.min(clampedRemaining, attemptCapMs, Math.max(1, attemptBudgetMs));
+}
 function filterNetworkSummaryRecords(records, input) {
   return records.filter((record) => {
     if (record.record.resourceType === "preflight" || record.record.method === "OPTIONS") {
@@ -21905,10 +22016,10 @@ function extractGraphqlOperationName(queryText) {
 function shouldShowRequestBody(method) {
   return !["GET", "HEAD", "DELETE", "OPTIONS"].includes(method.trim().toUpperCase());
 }
-function buildStructuredBodyPreview(body, headers) {
+function buildStructuredBodyPreview(body, headers, options = {}) {
   const contentType = headerValue(headers, "content-type") ?? body?.mimeType;
   const parsed = parseStructuredPayload(body, contentType);
-  const data = parsed === void 0 ? void 0 : typeof parsed === "string" ? truncateInlineText(parsed) : truncateStructuredValue(parsed);
+  const data = parsed === void 0 ? void 0 : options.truncateData === false ? parsed : typeof parsed === "string" ? truncateInlineText(parsed) : truncateStructuredValue(parsed);
   return {
     bytes: body?.originalByteLength ?? body?.capturedByteLength ?? 0,
     ...contentType === void 0 ? {} : { contentType },
@@ -22121,10 +22232,12 @@ function resolveSessionFetchTransportLadder(transport) {
       return ["direct-http"];
     case "matched-tls":
       return ["matched-tls"];
+    case "context":
+      return ["context-http"];
     case "page":
       return ["page-http"];
     case "auto":
-      return ["direct-http", "matched-tls", "page-http"];
+      return ["direct-http", "matched-tls", "context-http", "page-http"];
   }
 }
 function detectChallengeNoteFromRecord(record) {
@@ -22269,6 +22382,12 @@ function diffStorageSnapshot(left, right) {
 function normalizeRuntimeErrorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function normalizeProbeTransportAttemptError(transport, error, attemptTimeoutMs) {
+  if (attemptTimeoutMs !== void 0 && error instanceof OpensteerProtocolError && error.code === "timeout") {
+    return `${transport} probe exceeded ${String(attemptTimeoutMs)}ms`;
+  }
+  return normalizeRuntimeErrorMessage(error);
+}
 function applyBrowserCookiesToTransportRequest(request, cookies) {
   if (cookies.length === 0) {
     return request;
@@ -24803,9 +24922,11 @@ exports.createFilesystemOpensteerWorkspace = createFilesystemOpensteerWorkspace;
 exports.createFlowRecorderInstallScript = createFlowRecorderInstallScript;
 exports.createObservationStore = createObservationStore;
 exports.createOpensteerExtractionDescriptorStore = createOpensteerExtractionDescriptorStore;
+exports.createSavedNetworkStore = createSavedNetworkStore;
 exports.dispatchSemanticOperation = dispatchSemanticOperation;
 exports.generateReplayScript = generateReplayScript;
 exports.hashDomDescriptorPersist = hashDomDescriptorPersist;
+exports.iterateSavedNetworkRecordBatches = iterateSavedNetworkRecordBatches;
 exports.manifestToExternalBinaryLocation = manifestToExternalBinaryLocation;
 exports.normalizeObservabilityConfig = normalizeObservabilityConfig;
 exports.normalizeWorkspaceId = normalizeWorkspaceId;