backtrace-console 0.0.3 → 0.0.4

package/lib/backtrace/query.js

@@ -0,0 +1,506 @@
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { AuthenticationError, BacktraceSession, buildSiblingUrl, buildLoginUrl, withSessionToken } = require("./query-session");
+const { downloadAttachment, downloadAttachmentWithRetry } = require("./query-download");
+const { logStep, ensureDirectory, normalizeFingerprint, getFingerprintLogsRoot } = require("./utils");
+let undiciModule = null;
+const FINGERPRINT_GROUP_FIELDS = ["fingerprint"];
+const FINGERPRINT_GROUP_FOLD = { "error.message": [["head"]], classifiers: [["head"]], "fingerprint;first_seen": [["head"]], "fingerprint;issues;state": [["head"]] };
+const FINGERPRINT_GROUP_ORDER = [{ name: "fingerprint;first_seen;0", ordering: "descending" }];
+function getUndici() {
+  if (!undiciModule) {
+    undiciModule = require("undici");
+  }
+  return undiciModule;
+}
+function createProxyDispatcher(proxyUrl) {
+  const normalizedProxy = String(proxyUrl || "").trim();
+  if (!normalizedProxy) {
+    return null;
+  }
+  try {
+    const { ProxyAgent } = getUndici();
+    return new ProxyAgent(normalizedProxy);
+  } catch (error) {
+    throw new Error(`Proxy support requires dependency "undici": ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+function withDispatcher(options, dispatcher) {
+  if (!dispatcher) {
+    return options || {};
+  }
+  return { ...(options || {}), dispatcher };
+}
+function shouldUseFingerprintGroupQuery(options) { return options.command === "fingerprint"; }
+function getRequestedFingerprints(options) {
+  return String(options?.fingerprint || "")
+    .split(",")
+    .map((item) => String(item || "").trim())
+    .filter(Boolean);
+}
+function buildQueryBody(options, offset, limit) {
+  const filter = {
+    timestamp: [["at-most", String(options.to)], ["at-least", String(options.from)]],
+  };
+  if (options.fingerprint) {
+    filter.fingerprint = [["contains", String(options.fingerprint)]];
+  }
+  if (shouldUseFingerprintGroupQuery(options)) {
+    return {
+      group: FINGERPRINT_GROUP_FIELDS,
+      fold: FINGERPRINT_GROUP_FOLD,
+      order: FINGERPRINT_GROUP_ORDER,
+      offset,
+      limit,
+      filter: [filter],
+    };
+  }
+  return {
+    select: options.select,
+    order: [{ name: "timestamp", ordering: "descending" }],
+    offset,
+    limit,
+    filter: [filter],
+  };
+}
+function buildFingerprintGroupQueryBody(options, offset, limit, fingerprintFilter) {
+  const filter = {
+    timestamp: [["at-most", String(options.to)], ["at-least", String(options.from)]],
+  };
+  if (fingerprintFilter) {
+    filter.fingerprint = [["contains", String(fingerprintFilter)]];
+  }
+  return {
+    group: FINGERPRINT_GROUP_FIELDS,
+    fold: FINGERPRINT_GROUP_FOLD,
+    order: FINGERPRINT_GROUP_ORDER,
+    offset,
+    limit,
+    filter: [filter],
+  };
+}
+function buildCompressedFingerprintQueryBody(options, fingerprint) {
+  return {
+    select: ["_compressed"],
+    order: [{ name: "timestamp", ordering: "descending" }],
+    offset: 0,
+    limit: 1,
+    filter: [{
+      fingerprint: [["contains", String(fingerprint)]],
+      timestamp: [["at-most", String(options.to)], ["at-least", String(options.from)]],
+    }],
+  };
+}
+function buildFingerprintObjectsQueryBody(options, fingerprint, offset, limit) {
+  return {
+    select: ["timestamp"],
+    order: [{ name: "timestamp", ordering: "descending" }],
+    offset,
+    limit,
+    filter: [{
+      fingerprint: [["contains", String(fingerprint)]],
+      timestamp: [["at-most", String(options.to)], ["at-least", String(options.from)]],
+    }],
+  };
+}
+function toHexObjectId(value) {
+  if (typeof value === "string" && /^[0-9a-f]+$/i.test(value)) {
+    return value.toLowerCase();
+  }
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric)) {
+    return String(value);
+  }
+  return numeric.toString(16);
+}
+function extractObjectIds(objects) {
+  if (!Array.isArray(objects) || objects.length === 0) {
+    return [];
+  }
+  const values = [];
+  objects.forEach((group) => {
+    const [, rows] = group || [];
+    if (!Array.isArray(rows)) {
+      return;
+    }
+    rows.forEach((row) => {
+      const value = Array.isArray(row) ? row[0] : null;
+      if (value !== null && value !== undefined) {
+        values.push(value);
+      }
+    });
+  });
+  return values;
+}
+function extractSelectedValues(values, selectFields) {
+  if (!Array.isArray(values) || values.length === 0) {
+    return [];
+  }
+  const [, ...entries] = values[0];
+  return entries.map((entry) => {
+    const row = {};
+    selectFields.forEach((field, index) => {
+      row[field] = Array.isArray(entry) ? entry[index] : undefined;
+    });
+    if (Array.isArray(entry) && entry.length > selectFields.length) {
+      row._count = entry[entry.length - 1];
+    }
+    return row;
+  });
+}
+function mapResultsToItems(objectIds, selectedValues) {
+  return objectIds.map((objectId, index) => ({
+    index: index + 1,
+    objectIdDecimal: objectId,
+    objectIdHex: toHexObjectId(objectId),
+    values: selectedValues[index] || {},
+  }));
+}
+function optimizeLimit(totalRows, baseLimit) { if (totalRows <= baseLimit) return baseLimit; if (totalRows <= 100) return 50; if (totalRows <= 500) return 100; return 200; }
+function unwrapValueCell(value) {
+  if (Array.isArray(value) && value.length === 1) {
+    return unwrapValueCell(value[0]);
+  }
+  return value;
+}
+function normalizeFieldName(value) {
+  return String(value || "")
+    .replace(/^head\((.+)\)$/i, "$1")
+    .replace(/;0$/i, "");
+}
+function extractGroupedFingerprintItems(response) {
+  const values = response?.values;
+  if (!Array.isArray(values) || values.length === 0) {
+    return [];
+  }
+  const rows = values;
+  const valueHeaders = Array.isArray(response?.columns_desc) && response.columns_desc.length > 0
+    ? response.columns_desc.map((entry) => normalizeFieldName(entry?.name))
+    : Array.isArray(response?.columns)
+      ? response.columns.map((entry) => normalizeFieldName(Array.isArray(entry) ? entry[0] : entry))
+      : [];
+  return rows.map((entry, index) => {
+    const rowValues = Array.isArray(entry) ? entry : [];
+    const fingerprint = unwrapValueCell(rowValues[0]);
+    const groupedColumns = Array.isArray(rowValues[1]) ? rowValues[1] : [];
+    const count = rowValues[2];
+    const mappedValues = {};
+    mappedValues.fingerprint = String(fingerprint || "").trim();
+    valueHeaders.forEach((field, fieldIndex) => {
+      mappedValues[field] = unwrapValueCell(groupedColumns[fieldIndex]);
+    });
+    if (count !== undefined) {
+      mappedValues._count = unwrapValueCell(count);
+    }
+    return {
+      index: index + 1,
+      objectIdDecimal: "",
+      objectIdHex: "",
+      fingerprint: mappedValues.fingerprint,
+      values: mappedValues,
+    };
+  });
+}
+async function fetchBacktracePage(session, queryUrl, queryBody) {
+  logStep("query", "sending query page", queryBody);
+  return session.requestJson(
+    queryUrl,
+    { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(queryBody) },
+    `query offset=${queryBody.offset} limit=${queryBody.limit}`,
+  );
+}
+async function fetchFingerprintProbe(session, options, fingerprint) {
+  const queryBody = buildCompressedFingerprintQueryBody(options, fingerprint);
+  const result = await fetchBacktracePage(session, options.queryUrl, queryBody);
+  return {
+    fingerprint,
+    totalRows: result?._?.runtime?.filter?.rows ?? 0,
+    payload: result,
+  };
+}
+async function queryFingerprintGroups(session, options, fingerprintFilter = "") {
+  const probeBody = buildFingerprintGroupQueryBody(options, options.offset, options.limit, fingerprintFilter);
+  const probeResult = await fetchBacktracePage(session, options.queryUrl, probeBody);
+  const totalRows = probeResult?.response?.cardinalities?.pagination?.groups
+    ?? probeResult?.response?.cardinalities?.having?.groups
+    ?? probeResult?.response?.cardinalities?.initial?.groups
+    ?? 0;
+  const pageLimit = optimizeLimit(totalRows, options.limit);
+  logStep("query", "fingerprint pagination decided", {
+    totalRows,
+    initialLimit: options.limit,
+    optimizedLimit: pageLimit,
+    fingerprint: fingerprintFilter || "all",
+  });
+  const items = [];
+  for (let offset = options.offset; offset < totalRows + options.offset || (offset === options.offset && totalRows === 0); offset += pageLimit) {
+    /* eslint-disable no-await-in-loop */
+    const result = await fetchBacktracePage(
+      session,
+      options.queryUrl,
+      buildFingerprintGroupQueryBody(options, offset, pageLimit, fingerprintFilter),
+    );
+    /* eslint-enable no-await-in-loop */
+    items.push(...extractGroupedFingerprintItems(result?.response));
+    if (totalRows === 0) {
+      break;
+    }
+  }
+  return {
+    totalRows,
+    pageLimit,
+    listedCount: items.length,
+    objectIds: items.map((item) => item.fingerprint || ""),
+    selectedValues: items.map((item) => item.values || {}),
+    items,
+  };
+}
+async function fetchFingerprintObjects(session, options, fingerprint, knownTotalRows) {
+  const totalRows = Number.isFinite(knownTotalRows)
+    ? knownTotalRows
+    : (() => {
+      throw new Error("fetchFingerprintObjects requires a known total row count");
+    })();
+  const pageLimit = optimizeLimit(totalRows, options.limit);
+  const objects = [];
+  for (let offset = options.offset; offset < totalRows + options.offset || (offset === options.offset && totalRows === 0); offset += pageLimit) {
+    /* eslint-disable no-await-in-loop */
+    const result = await fetchBacktracePage(
+      session,
+      options.queryUrl,
+      buildFingerprintObjectsQueryBody(options, fingerprint, offset, pageLimit),
+    );
+    /* eslint-enable no-await-in-loop */
+    const objectIds = extractObjectIds(result?.response?.objects);
+    const selectedValues = extractSelectedValues(result?.response?.values, ["timestamp"]);
+    objectIds.forEach((objectId, index) => {
+      objects.push({
+        objectIdDecimal: objectId,
+        objectIdHex: toHexObjectId(objectId),
+        timestamp: selectedValues[index]?.timestamp || "",
+      });
+    });
+    if (totalRows === 0) {
+      break;
+    }
+  }
+  return {
+    fingerprint,
+    totalRows,
+    pageLimit,
+    objects,
+  };
+}
+async function queryFingerprintDetail(session, options) {
+  const requestedFingerprints = getRequestedFingerprints(options);
+  const fingerprint = requestedFingerprints[0];
+  const groupResult = await queryFingerprintGroups(session, options, fingerprint);
+  const summary = groupResult.items[0] || {
+    fingerprint,
+    values: {
+      fingerprint,
+      "error.message": "",
+      classifiers: "",
+      "fingerprint;first_seen": "",
+      "fingerprint;issues;state": "",
+      _count: 0,
+    },
+  };
+  const probe = await fetchFingerprintProbe(session, options, fingerprint);
+  const objectsResult = await fetchFingerprintObjects(session, options, fingerprint, probe.totalRows);
+  return {
+    fingerprint,
+    summary,
+    objects: objectsResult.objects,
+    querySummary: {
+      totalRows: objectsResult.totalRows,
+      fetchedRows: objectsResult.objects.length,
+      objectCount: objectsResult.objects.length,
+      pageLimit: objectsResult.pageLimit,
+      objectIds: objectsResult.objects.map((item) => item.objectIdHex),
+      selectedValues: objectsResult.objects.map((item) => ({ timestamp: item.timestamp })),
+    },
+  };
+}
+async function fetchAllResults(session, options) {
+  const probeBody = buildQueryBody(options, options.offset, options.limit);
+  const probeResult = await fetchBacktracePage(session, options.queryUrl, probeBody);
+  const totalRows = shouldUseFingerprintGroupQuery(options)
+    ? probeResult?.response?.cardinalities?.pagination?.groups
+      ?? probeResult?.response?.cardinalities?.having?.groups
+      ?? probeResult?.response?.cardinalities?.initial?.groups
+      ?? 0
+    : probeResult?._?.runtime?.filter?.rows ?? 0;
+  const pageLimit = optimizeLimit(totalRows, options.limit);
+  logStep("query", "query pagination decided", { totalRows, initialLimit: options.limit, optimizedLimit: pageLimit });
+  const pages = [];
+  for (let offset = options.offset; offset < totalRows + options.offset || (offset === options.offset && totalRows === 0); offset += pageLimit) {
+    pages.push({ offset, limit: pageLimit });
+    if (totalRows === 0) break;
+  }
+  if (pages.length === 0) {
+    pages.push({ offset: options.offset, limit: pageLimit });
+  }
+  const allObjectIds = [];
+  const allSelectedValues = [];
+  for (const page of pages) {
+    /* eslint-disable no-await-in-loop */
+    const result = await fetchBacktracePage(session, options.queryUrl, buildQueryBody(options, page.offset, page.limit));
+    /* eslint-enable no-await-in-loop */
+    if (shouldUseFingerprintGroupQuery(options)) {
+      const groupedItems = extractGroupedFingerprintItems(result?.response);
+      allObjectIds.push(...groupedItems.map((item) => item.fingerprint || ""));
+      allSelectedValues.push(...groupedItems.map((item) => item.values));
+    } else {
+      allObjectIds.push(...extractObjectIds(result?.response?.objects));
+      allSelectedValues.push(...extractSelectedValues(result?.response?.values, options.select));
+    }
+  }
+  return { totalRows, pageLimit, listedCount: allObjectIds.length, objectIds: allObjectIds, selectedValues: allSelectedValues };
+}
+async function fetchAttachmentList(session, queryUrl, objectHexId) {
+  const source = new URL(queryUrl);
+  const url = buildSiblingUrl(queryUrl, "/api/list");
+  const project = source.searchParams.get("project");
+  if (project) url.searchParams.set("project", project);
+  url.searchParams.set("object", objectHexId);
+  logStep("attachments", "requesting attachment list", url.toString());
+  return session.requestJson(url.toString(), { method: "GET" }, `attachment list ${objectHexId}`);
+}
+async function queryAllItems(session, options) {
+  if (options.command === "collect-all") {
+    const requestedFingerprints = getRequestedFingerprints(options);
+    const fingerprintItems = [];
+    let fingerprintPageLimit = 0;
+    if (requestedFingerprints.length === 0) {
+      const fingerprintResult = await queryFingerprintGroups(session, options);
+      fingerprintItems.push(...fingerprintResult.items);
+      fingerprintPageLimit = fingerprintResult.pageLimit;
+    } else {
+      for (const requestedFingerprint of requestedFingerprints) {
+        /* eslint-disable no-await-in-loop */
+        const fingerprintResult = await queryFingerprintGroups(session, options, requestedFingerprint);
+        fingerprintItems.push(...fingerprintResult.items);
+        fingerprintPageLimit = Math.max(fingerprintPageLimit, fingerprintResult.pageLimit);
+        /* eslint-enable no-await-in-loop */
+      }
+    }
+    const uniqueFingerprintItems = Array.from(
+      new Map(fingerprintItems.map((item) => [String(item.fingerprint || item.values.fingerprint || "").trim(), item])).values(),
+    ).filter((item) => String(item.fingerprint || item.values.fingerprint || "").trim());
+    const items = [];
+    let totalRows = 0;
+    let pageLimit = fingerprintPageLimit;
+    for (const fingerprintItem of uniqueFingerprintItems) {
+      const fingerprint = String(fingerprintItem.fingerprint || fingerprintItem.values.fingerprint || "").trim();
+      if (!fingerprint) {
+        continue;
+      }
+      /* eslint-disable no-await-in-loop */
+      const probe = await fetchFingerprintProbe(session, options, fingerprint);
+      if (probe.totalRows === 0) {
+        logStep("query", "fingerprint probe returned no rows", { fingerprint });
+        continue;
+      }
+      const objectResult = await fetchFingerprintObjects(session, options, fingerprint, probe.totalRows);
+      totalRows += objectResult.totalRows;
+      pageLimit = Math.max(pageLimit, objectResult.pageLimit);
+      objectResult.objects.forEach((objectItem) => {
+        items.push({
+          index: items.length + 1,
+          objectIdDecimal: objectItem.objectIdDecimal,
+          objectIdHex: objectItem.objectIdHex,
+          fingerprint,
+          values: {
+            fingerprint,
+            timestamp: objectItem.timestamp,
+            "error.message": fingerprintItem.values["error.message"] || "",
+            classifiers: fingerprintItem.values.classifiers || "",
+            "fingerprint;first_seen": fingerprintItem.values["fingerprint;first_seen"] || "",
+            "fingerprint;issues;state": fingerprintItem.values["fingerprint;issues;state"] || "",
+            _count: fingerprintItem.values._count || 0,
+          },
+        });
+      });
+      /* eslint-enable no-await-in-loop */
+    }
+    logStep("query", "loaded query results", {
+      fingerprintCount: uniqueFingerprintItems.length,
+      fetchedRows: totalRows,
+      objectCount: items.length,
+    });
+    return {
+      totalRows,
+      pageLimit,
+      listedCount: items.length,
+      objectIds: items.map((item) => item.objectIdHex),
+      selectedValues: items.map((item) => item.values),
+      items,
+      fingerprints: uniqueFingerprintItems,
+    };
+  }
+  const result = await fetchAllResults(session, options);
+  const items = result.selectedValues.map((values, index) => ({
+    index: index + 1,
+    objectIdDecimal: "",
+    objectIdHex: "",
+    fingerprint: String(values?.fingerprint || "").trim(),
+    values: values || {},
+  }));
+  logStep("query", "loaded query results", { fetchedRows: result.listedCount, objectCount: items.length });
+  return { ...result, items };
+}
+async function downloadItemLogs(session, item, options) {
+  const objectHexId = item.objectIdHex;
+  const fingerprint = normalizeFingerprint(item.fingerprint || item.values.fingerprint || options.fingerprint);
+  logStep("download", "preparing object download", { objectId: objectHexId, from: options.from, to: options.to, fingerprint });
+  let attachmentList = null;
+  let attachmentListError = null;
+  for (let attempt = 1; attempt <= 5; attempt += 1) {
+    try {
+      /* eslint-disable no-await-in-loop */
+      attachmentList = await fetchAttachmentList(session, options.queryUrl, objectHexId);
+      /* eslint-enable no-await-in-loop */
+      attachmentListError = null;
+      break;
+    } catch (error) {
+      attachmentListError = error;
+      logStep("attachments", "attachment list failed", {
+        objectId: objectHexId,
+        attempt,
+        maxAttempts: 5,
+        message: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  if (!attachmentList) {
+    logStep("download", "object skipped after attachment list retries", {
+      objectId: objectHexId,
+      fingerprint,
+      message: attachmentListError instanceof Error ? attachmentListError.message : String(attachmentListError || ""),
+    });
+    return { ...item, values: { ...item.values, fingerprint }, fingerprint, targetDir: "", downloadedFiles: [], skipped: true, failed: true };
+  }
+  const attachments = Array.isArray(attachmentList.attachments) ? attachmentList.attachments : [];
+  const targetDir = path.join(getFingerprintLogsRoot(options.storageDir, fingerprint), objectHexId);
+  await ensureDirectory(targetDir);
+  const downloadedFiles = [];
+  for (const attachment of attachments) {
+    /* eslint-disable no-await-in-loop */
+    const downloadedFile = await downloadAttachmentWithRetry(session, options.queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl, 5);
+    downloadedFiles.push(downloadedFile);
+    if (typeof options.onAttachmentDownloaded === "function") {
+      options.onAttachmentDownloaded({
+        item,
+        objectId: objectHexId,
+        fingerprint,
+        attachmentName: attachment.name,
+        downloadedFile,
+      });
+    }
+    /* eslint-enable no-await-in-loop */
+  }
+  logStep("download", "object download completed", { objectId: objectHexId, fileCount: downloadedFiles.length, targetDir, fingerprint });
+  return { ...item, values: { ...item.values, fingerprint }, fingerprint, targetDir, downloadedFiles };
+}
+module.exports = { AuthenticationError, BacktraceSession, buildQueryBody, buildSiblingUrl, buildLoginUrl, withSessionToken, toHexObjectId, extractObjectIds, extractSelectedValues, mapResultsToItems, optimizeLimit, fetchBacktracePage, fetchAllResults, queryFingerprintGroups, queryFingerprintDetail, fetchAttachmentList, downloadAttachment, queryAllItems, downloadItemLogs };