@rawdash/core 0.15.0 → 0.17.0

package/dist/index.js

@@ -291,8 +291,79 @@ function computeDelay(attempt, initialDelayMs, maxDelayMs) {
   const jitter = base * 0.25 * Math.random();
   return Math.min(base + jitter, maxDelayMs);
 }
+var MAX_VALUE_LEN = 120;
+function truncate(s, max = MAX_VALUE_LEN) {
+  if (s.length <= max) {
+    return s;
+  }
+  return `${s.slice(0, max - 1)}\u2026`;
+}
+function formatValue(value) {
+  if (value === null) {
+    return "null";
+  }
+  if (value === void 0) {
+    return "";
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  if (typeof value === "string") {
+    const t = truncate(value);
+    if (/[\s"=]/.test(t)) {
+      return JSON.stringify(t);
+    }
+    return t;
+  }
+  if (typeof value === "bigint") {
+    return value.toString();
+  }
+  let json;
+  try {
+    json = JSON.stringify(value);
+  } catch {
+    json = void 0;
+  }
+  return truncate(json ?? String(value));
+}
+function formatLogFields(fields) {
+  if (!fields) {
+    return "";
+  }
+  const parts = [];
+  for (const [k, v] of Object.entries(fields)) {
+    if (v === void 0) {
+      continue;
+    }
+    parts.push(`${k}=${formatValue(v)}`);
+  }
+  return parts.length > 0 ? ` ${parts.join(" ")}` : "";
+}
+function formatLogLine(scope, event, fields) {
+  return `[${scope}] ${event}${formatLogFields(fields)}`;
+}
+function createDefaultConnectorLogger(opts) {
+  return {
+    info(event, fields) {
+      console.info(formatLogLine(opts.scope, event, fields));
+    },
+    warn(event, fields) {
+      console.warn(formatLogLine(opts.scope, event, fields));
+    }
+  };
+}
+var NOOP_LOGGER = {
+  info() {
+  },
+  warn() {
+  }
+};
+function noopConnectorLogger() {
+  return NOOP_LOGGER;
+}
 
 // src/secrets.ts
+import { z } from "zod";
 function secret(name) {
   if (!/^[A-Z][A-Z0-9_]*$/.test(name)) {
     throw new Error(
@@ -304,10 +375,31 @@ function secret(name) {
 function isSecret(value) {
   return typeof value === "object" && value !== null && "$secret" in value && typeof value.$secret === "string";
 }
+var secretRefSchema = z.strictObject({
+  $secret: z.string()
+});
+function withSecretRef(schema) {
+  return z.union([schema, secretRefSchema]);
+}
 var EnvSecretsResolver = class {
   resolve(name) {
     const env = globalThis.process?.env;
-    return env?.[name];
+    const raw = env?.[name];
+    if (raw === void 0) {
+      return void 0;
+    }
+    if (raw.length === 0) {
+      return raw;
+    }
+    const first = raw.charCodeAt(0);
+    if (first !== 123 && first !== 91) {
+      return raw;
+    }
+    try {
+      return JSON.parse(raw);
+    } catch {
+      return raw;
+    }
   }
 };
 function extractSecretNames(value) {
@@ -364,6 +456,7 @@ var BaseConnector = class {
   creds;
   rawCredInput;
   ctx;
+  cachedLogger;
   constructor(settings, creds, ctx) {
     this.settings = settings;
     this.rawCredInput = creds;
@@ -373,6 +466,12 @@ var BaseConnector = class {
       this.ctx.secretsResolver ?? new EnvSecretsResolver()
     ) : {};
   }
+  get logger() {
+    if (!this.cachedLogger) {
+      this.cachedLogger = this.ctx.logger ?? createDefaultConnectorLogger({ scope: this.id });
+    }
+    return this.cachedLogger;
+  }
   request(req, opts) {
     return request(req, {
       resource: opts.resource,
@@ -405,6 +504,13 @@ var BaseConnector = class {
       { resource: opts.resource, requestId: opts.requestId }
     );
   }
+  isResourceEnabled(resource) {
+    const enabled = this.settings?.resources;
+    if (!enabled || enabled.length === 0) {
+      return true;
+    }
+    return enabled.includes(resource);
+  }
   serializeConfig() {
     const config = {
       ...this.settings
@@ -478,8 +584,44 @@ function defineConnector() {
 }
 
 // src/paginate-chunked.ts
+function selectActivePhases(resourceToPhase, order, enabled) {
+  if (!enabled || enabled.length === 0) {
+    return [...order];
+  }
+  const want = /* @__PURE__ */ new Set();
+  for (const r of enabled) {
+    want.add(resourceToPhase(r));
+  }
+  return order.filter((p) => want.has(p));
+}
+function makeChunkedCursorGuard(phases) {
+  const phaseSet = new Set(phases);
+  return (value) => {
+    if (typeof value !== "object" || value === null) {
+      return false;
+    }
+    const v = value;
+    if (typeof v.phase !== "string" || !phaseSet.has(v.phase)) {
+      return false;
+    }
+    if (v.page !== null && typeof v.page !== "string") {
+      return false;
+    }
+    return true;
+  };
+}
+function truncateCursor(page) {
+  if (page === null || page === void 0) {
+    return void 0;
+  }
+  const s = typeof page === "string" ? page : JSON.stringify(page);
+  if (s.length <= 80) {
+    return s;
+  }
+  return `${s.slice(0, 79)}\u2026`;
+}
 async function paginateChunked(opts) {
-  const { phases, cursor, signal, fetchPage, writeBatch } = opts;
+  const { phases, cursor, signal, fetchPage, writeBatch, logger } = opts;
   if (phases.length === 0) {
     return { done: true };
   }
@@ -489,6 +631,9 @@ async function paginateChunked(opts) {
   for (let i = startIdx; i < phases.length; i++) {
     const phase = phases[i];
     let page = i === startIdx && hasKnownResumePhase ? cursor.page : null;
+    let pageCount = 0;
+    let itemCount = 0;
+    const phaseStart = Date.now();
     while (true) {
       if (signal?.aborted) {
         return {
@@ -496,6 +641,7 @@ async function paginateChunked(opts) {
           cursor: { phase, page }
         };
       }
+      pageCount += 1;
       let items;
       let next;
       try {
@@ -507,12 +653,26 @@ async function paginateChunked(opts) {
             cursor: { phase, page }
           };
         }
+        logger?.warn("fetch page failed", {
+          resource: phase,
+          page: pageCount,
+          cursor: truncateCursor(page),
+          error: err instanceof Error ? err.message : String(err)
+        });
         return {
           done: false,
           cursor: { phase, page },
           transientError: err
         };
       }
+      itemCount += items.length;
+      logger?.info("fetched page", {
+        resource: phase,
+        page: pageCount,
+        items: items.length,
+        cursor: truncateCursor(page),
+        next: truncateCursor(next)
+      });
       try {
         await writeBatch(phase, items, page);
       } catch (err) {
@@ -522,6 +682,12 @@ async function paginateChunked(opts) {
             cursor: { phase, page }
           };
         }
+        logger?.warn("write batch failed", {
+          resource: phase,
+          page: pageCount,
+          cursor: truncateCursor(page),
+          error: err instanceof Error ? err.message : String(err)
+        });
         return {
           done: false,
           cursor: { phase, page },
@@ -533,20 +699,26 @@ async function paginateChunked(opts) {
       }
       page = next;
     }
+    logger?.info("resource done", {
+      resource: phase,
+      pages: pageCount,
+      items: itemCount,
+      duration_ms: Date.now() - phaseStart
+    });
   }
   return { done: true };
 }
 
 // src/widget-schemas.ts
-import { z } from "zod";
-var shapeSchema = z.enum([
+import { z as z2 } from "zod";
+var shapeSchema = z2.enum([
   "event",
   "entity",
   "metric",
   "edge",
   "distribution"
 ]);
-var aggFnSchema = z.enum([
+var aggFnSchema = z2.enum([
   "count",
   "sum",
   "avg",
@@ -555,7 +727,7 @@ var aggFnSchema = z.enum([
   "latest",
   "first"
 ]);
-var filterOperatorSchema = z.enum([
+var filterOperatorSchema = z2.enum([
   "eq",
   "neq",
   "gt",
@@ -564,70 +736,70 @@ var filterOperatorSchema = z.enum([
   "lte",
   "contains"
 ]);
-var filterConditionSchema = z.object({
-  field: z.string(),
+var filterConditionSchema = z2.object({
+  field: z2.string(),
   op: filterOperatorSchema,
-  value: z.union([z.string(), z.number(), z.boolean()])
+  value: z2.union([z2.string(), z2.number(), z2.boolean()])
 });
-var filterClauseSchema = z.union([
+var filterClauseSchema = z2.union([
   filterConditionSchema,
-  z.object({ or: z.array(filterConditionSchema) })
+  z2.object({ or: z2.array(filterConditionSchema) })
 ]);
-var groupBySchema = z.object({
-  field: z.string(),
-  granularity: z.enum(["hour", "day", "week", "month"])
+var groupBySchema = z2.object({
+  field: z2.string(),
+  granularity: z2.enum(["hour", "day", "week", "month"])
 });
-var computedMetricSchema = z.object({
-  connectorId: z.string(),
+var computedMetricSchema = z2.object({
+  connectorId: z2.string(),
   shape: shapeSchema,
-  name: z.string().optional(),
-  entityType: z.string().optional(),
-  field: z.string().optional(),
+  name: z2.string().optional(),
+  entityType: z2.string().optional(),
+  field: z2.string().optional(),
   fn: aggFnSchema,
-  window: z.string().optional(),
-  filter: z.array(filterClauseSchema).optional(),
+  window: z2.string().optional(),
+  filter: z2.array(filterClauseSchema).optional(),
   groupBy: groupBySchema.optional()
 }).refine((m) => m.fn === "count" || m.field !== void 0, {
   message: 'field is required unless fn is "count"',
   path: ["field"]
 });
-var titleField = z.string().meta({ label: "Title", description: "Widget title." });
-var statWidgetSchema = z.object({
-  kind: z.literal("stat"),
+var titleField = z2.string().meta({ label: "Title", description: "Widget title." });
+var statWidgetSchema = z2.object({
+  kind: z2.literal("stat"),
   title: titleField,
   metric: computedMetricSchema.meta({
     label: "Metric",
     description: "Computed metric definition."
   }),
-  window: z.string().optional().meta({ label: "Window", description: "Time window, e.g. '7d'." }),
-  compare: z.enum(["none", "previous-period"]).default("none").meta({ label: "Compare", description: "Comparison mode." })
+  window: z2.string().optional().meta({ label: "Window", description: "Time window, e.g. '7d'." }),
+  compare: z2.enum(["none", "previous-period"]).default("none").meta({ label: "Compare", description: "Comparison mode." })
 });
-var statusWidgetSchema = z.object({
-  kind: z.literal("status"),
+var statusWidgetSchema = z2.object({
+  kind: z2.literal("status"),
   title: titleField,
-  source: z.string().meta({
+  source: z2.string().meta({
     label: "Source",
     description: "Connector or data source reference."
   })
 });
-var timeseriesWidgetSchema = z.object({
-  kind: z.literal("timeseries"),
+var timeseriesWidgetSchema = z2.object({
+  kind: z2.literal("timeseries"),
   title: titleField,
   metric: computedMetricSchema.meta({
     label: "Metric",
     description: "Computed metric definition."
   }),
-  window: z.string().meta({ label: "Window", description: "Time window, e.g. '30d'." }),
-  granularity: z.enum(["hour", "day", "week"]).default("day").meta({ label: "Granularity", description: "Time bucket size." })
+  window: z2.string().meta({ label: "Window", description: "Time window, e.g. '30d'." }),
+  granularity: z2.enum(["hour", "day", "week"]).default("day").meta({ label: "Granularity", description: "Time bucket size." })
 });
-var distributionWidgetSchema = z.object({
-  kind: z.literal("distribution"),
+var distributionWidgetSchema = z2.object({
+  kind: z2.literal("distribution"),
   title: titleField,
   metric: computedMetricSchema.meta({
     label: "Metric",
     description: "Computed metric definition."
   }),
-  window: z.string().meta({ label: "Window", description: "Time window, e.g. '7d'." })
+  window: z2.string().meta({ label: "Window", description: "Time window, e.g. '7d'." })
 });
 var widgetSchemas = {
   stat: statWidgetSchema,
@@ -635,7 +807,7 @@ var widgetSchemas = {
   timeseries: timeseriesWidgetSchema,
   distribution: distributionWidgetSchema
 };
-var widgetSchema = z.discriminatedUnion("kind", [
+var widgetSchema = z2.discriminatedUnion("kind", [
   statWidgetSchema,
   statusWidgetSchema,
   timeseriesWidgetSchema,
@@ -837,9 +1009,9 @@ async function computeRetention(handle, config, nowMs = Date.now()) {
 }
 
 // src/config-fields.ts
-import { z as z2 } from "zod";
+import { z as z3 } from "zod";
 function defineConfigFields(schema) {
-  if (!(schema instanceof z2.ZodObject)) {
+  if (!(schema instanceof z3.ZodObject)) {
     throw new Error(
       `configFields must be a Zod object schema (z.object({...})). Received: ${Object.prototype.toString.call(schema)}`
     );
@@ -847,6 +1019,88 @@ function defineConfigFields(schema) {
   return schema;
 }
 
+// src/connector-doc.ts
+import { z as z4 } from "zod";
+var connectorCategorySchema = z4.enum([
+  "engineering",
+  "product",
+  "analytics",
+  "marketing",
+  "sales",
+  "support",
+  "finance",
+  "infrastructure",
+  "security"
+]);
+var connectorDocSchema = z4.object({
+  displayName: z4.string().min(1),
+  category: connectorCategorySchema,
+  tagline: z4.string().min(1),
+  // Brand accent color (hex) for the connector's docs/landing card. The icon
+  // itself is a committed `icon.svg` co-located in the connector package.
+  brandColor: z4.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),
+  vendor: z4.object({
+    name: z4.string().min(1),
+    apiDocs: z4.url().optional(),
+    website: z4.url().optional()
+  }),
+  auth: z4.object({
+    summary: z4.string().min(1),
+    setup: z4.array(z4.string().min(1))
+  }),
+  // Upstream API rate-limit / quota notes worth surfacing (e.g. "GA4 Data API:
+  // 200k tokens/day per property"). Free text, rendered in docs.
+  rateLimit: z4.string().min(1).optional(),
+  // Operational caveats / out-of-scope notes (API ceilings, sampling, Cloud-only,
+  // data revision windows, etc.).
+  limitations: z4.array(z4.string().min(1)).optional()
+});
+function defineConnectorDoc(doc) {
+  return connectorDocSchema.parse(doc);
+}
+
+// src/resource.ts
+var SHAPES = /* @__PURE__ */ new Set([
+  "entity",
+  "event",
+  "metric",
+  "edge",
+  "distribution"
+]);
+function defineResources(defs) {
+  for (const [name, def] of Object.entries(defs)) {
+    if (!name) {
+      throw new Error("Resource name must be a non-empty string.");
+    }
+    if (!SHAPES.has(def.shape)) {
+      throw new Error(
+        `Resource "${name}" has invalid shape "${def.shape}". Expected one of: ${[...SHAPES].join(", ")}.`
+      );
+    }
+    if (!def.description || def.description.trim().length === 0) {
+      throw new Error(`Resource "${name}" must have a non-empty description.`);
+    }
+  }
+  return defs;
+}
+function schemasFromResources(defs) {
+  const out = {};
+  for (const [name, def] of Object.entries(defs)) {
+    if (!def.responses) {
+      continue;
+    }
+    for (const [tag, schema] of Object.entries(def.responses)) {
+      if (out[tag]) {
+        throw new Error(
+          `Duplicate response schema tag "${tag}" (declared again on resource "${name}").`
+        );
+      }
+      out[tag] = schema;
+    }
+  }
+  return out;
+}
+
 // src/compute.ts
 function matchesCondition(record, cond) {
   const val = record[cond.field];
@@ -1101,6 +1355,77 @@ async function computeMetric(storage, metric) {
   return computeAgg(sorted, metric.field, metric.fn);
 }
 
+// src/backfill-window.ts
+var WINDOW_UNIT_MS = {
+  h: 36e5,
+  d: 864e5,
+  w: 6048e5,
+  m: 2592e6
+};
+function parseWindowMs2(window) {
+  const match = /^(\d+)(h|d|w|m)$/.exec(window);
+  if (!match) {
+    return void 0;
+  }
+  const unitMs = WINDOW_UNIT_MS[match[2]];
+  if (unitMs === void 0) {
+    return void 0;
+  }
+  return parseInt(match[1]) * unitMs;
+}
+function widgetWindow(widget) {
+  switch (widget.kind) {
+    case "stat":
+      return widget.window ?? widget.metric.window;
+    case "timeseries":
+    case "distribution":
+      return widget.window;
+    case "status":
+      return void 0;
+  }
+}
+function widgetReference(widget) {
+  if (widget.kind === "status") {
+    return { connectorName: widget.source, resourceName: void 0 };
+  }
+  return {
+    connectorName: widget.metric.connectorId,
+    resourceName: widget.metric.name ?? widget.metric.entityType
+  };
+}
+function mergeWindow(existing, next) {
+  if (next === void 0) {
+    return existing;
+  }
+  if (existing === void 0) {
+    return next;
+  }
+  return Math.max(existing, next);
+}
+function computeConnectorBackfill(config) {
+  const result = /* @__PURE__ */ new Map();
+  for (const dashboard of Object.values(config.dashboards)) {
+    for (const widget of Object.values(dashboard.widgets)) {
+      const { connectorName, resourceName } = widgetReference(widget);
+      const windowStr = widgetWindow(widget);
+      const windowMs = windowStr ? parseWindowMs2(windowStr) : void 0;
+      let resources = result.get(connectorName);
+      if (!resources) {
+        resources = /* @__PURE__ */ new Map();
+        result.set(connectorName, resources);
+      }
+      if (resourceName === void 0) {
+        continue;
+      }
+      const existing = resources.get(resourceName);
+      resources.set(resourceName, {
+        requiredWindowMs: mergeWindow(existing?.requiredWindowMs, windowMs)
+      });
+    }
+  }
+  return result;
+}
+
 // src/resolve-widget.ts
 var FAILING_CONNECTOR_STATUSES = /* @__PURE__ */ new Set(["error", "auth_failed", "paused"]);
 function deriveSyncStateFromHealth(health) {
@@ -1124,14 +1449,17 @@ function buildMetaFromHealth(health) {
   }
   return meta;
 }
-async function resolveWidget(widgetId, widget, connectors, storage) {
+async function resolveWidget(dashboardId, widgetId, widget, connectors, storage) {
   const connectorId = widget.kind === "status" ? widget.source : widget.metric.connectorId;
   if (connectors !== void 0 && !connectors.includes(connectorId)) {
     return void 0;
   }
   const handle = storage.getStorageHandle(connectorId);
   const health = await handle.getHealth?.() ?? null;
-  const data = widget.kind === "status" ? null : await computeMetric(handle, widget.metric);
+  let data = null;
+  if (widget.kind !== "status") {
+    data = await computeMetric(handle, widget.metric);
+  }
   let syncState;
   let meta;
   if (health) {
@@ -1148,12 +1476,40 @@ async function resolveWidget(widgetId, widget, connectors, storage) {
     data,
     cachedAt: health?.lastSyncAt ?? null,
     syncState,
+    syncIntervalSeconds: health?.syncIntervalSeconds,
     meta
   };
 }
 
+// src/widget-etag.ts
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value) ?? "null";
+  }
+  if (Array.isArray(value)) {
+    return "[" + value.map(stableStringify).join(",") + "]";
+  }
+  const keys = Object.keys(value).sort();
+  const parts = keys.map(
+    (k) => JSON.stringify(k) + ":" + stableStringify(value[k])
+  );
+  return "{" + parts.join(",") + "}";
+}
+function hashWidgetConfig(widget) {
+  const s = stableStringify(widget);
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+function computeWidgetEtag(lastSyncAt, widget) {
+  return `"${lastSyncAt ?? "null"}-${hashWidgetConfig(widget)}"`;
+}
+
 // src/registry.ts
-function instantiateConnector(entry, registry, secretsResolver) {
+function instantiateConnector(entry, registry, secretsResolver, logger) {
   const Cls = registry[entry.connectorId];
   if (!Cls) {
     throw new Error(
@@ -1171,7 +1527,8 @@ function instantiateConnector(entry, registry, secretsResolver) {
     }
   }
   return new Cls(settings, credSchema ? creds : void 0, {
-    secretsResolver
+    secretsResolver,
+    logger
   });
 }
 
@@ -1555,24 +1912,24 @@ var InMemoryStorage = class {
 };
 
 // src/wire-config.ts
-import { z as z3 } from "zod";
-var wireConnectorSchema = z3.object({
-  name: z3.string(),
-  connectorId: z3.string(),
-  displayName: z3.string().optional(),
-  config: z3.record(z3.string(), z3.unknown()),
-  syncIntervalSeconds: z3.number().optional(),
-  enabled: z3.boolean().optional()
+import { z as z5 } from "zod";
+var wireConnectorSchema = z5.object({
+  name: z5.string(),
+  connectorId: z5.string(),
+  displayName: z5.string().optional(),
+  config: z5.record(z5.string(), z5.unknown()),
+  syncIntervalSeconds: z5.number().optional(),
+  enabled: z5.boolean().optional()
 });
-var wireDashboardSchema = z3.object({
-  id: z3.string().optional(),
-  name: z3.string(),
-  slug: z3.string(),
-  config: z3.record(z3.string(), z3.unknown())
+var wireDashboardSchema = z5.object({
+  id: z5.string().optional(),
+  name: z5.string(),
+  slug: z5.string(),
+  config: z5.record(z5.string(), z5.unknown())
 });
-var wireConfigSchema = z3.object({
-  connectors: z3.array(wireConnectorSchema).optional(),
-  dashboards: z3.array(wireDashboardSchema).optional()
+var wireConfigSchema = z5.object({
+  connectors: z5.array(wireConnectorSchema).optional(),
+  dashboards: z5.array(wireDashboardSchema).optional()
 });
 function toWireConfig(config) {
   return {
@@ -1598,14 +1955,21 @@ export {
   EnvSecretsResolver,
   InMemoryStorage,
   aggFnSchema,
+  computeConnectorBackfill,
   computeMetric,
   computeRetention,
+  computeWidgetEtag,
   computedMetricSchema,
+  connectorCategorySchema,
+  connectorDocSchema,
+  createDefaultConnectorLogger,
   defineConfig,
   defineConfigFields,
   defineConnector,
+  defineConnectorDoc,
   defineDashboard,
   defineMetric,
+  defineResources,
   distributionWidgetSchema,
   extractSecretNames,
   filterClauseSchema,
@@ -1613,13 +1977,19 @@ export {
   filterOperatorSchema,
   getWidgetSchema,
   groupBySchema,
+  hashWidgetConfig,
   instantiateConnector,
   isSecret,
   isSyncActive,
+  makeChunkedCursorGuard,
+  noopConnectorLogger,
   paginateChunked,
   resolveSecrets,
   resolveWidget,
+  schemasFromResources,
   secret,
+  secretRefSchema,
+  selectActivePhases,
   selectForDeletion,
   shapeSchema,
   statWidgetSchema,
@@ -1631,6 +2001,7 @@ export {
   wireConfigSchema,
   wireConnectorSchema,
   wireDashboardSchema,
-  withAbortSignal
+  withAbortSignal,
+  withSecretRef
 };
 //# sourceMappingURL=index.js.map