@rawdash/core 0.14.0 → 0.16.0

package/README.md

@@ -9,7 +9,7 @@ Headless dashboard backend primitives for rawdash. Define connectors, dashboards
 
 `@rawdash/core` is the foundation of the rawdash ecosystem. It provides the types and functions (`defineConfig`, `defineDashboard`, `defineMetric`, `defineConnector`, `secret`) that every rawdash setup is built on. It has no framework dependencies and no I/O — it only models your dashboard configuration and the connector interface.
 
-Other packages (`@rawdash/server`, `@rawdash/hono`, `@rawdash/nextjs`, `@rawdash/mcp`) take a config produced by this package and add runtime behavior.
+Other packages (`@rawdash/server`, `@rawdash/hono`, `@rawdash/sdk-nextjs`, `@rawdash/mcp`) take a config produced by this package and add runtime behavior.
 
 ## Install
 
@@ -20,7 +20,6 @@ npm install @rawdash/core @rawdash/connector-github
 ## Quick example
 
 ```ts
-import { GitHubConnector } from '@rawdash/connector-github';
 import {
   defineConfig,
   defineDashboard,
@@ -28,18 +27,18 @@ import {
   secret,
 } from '@rawdash/core';
 
-const github = new GitHubConnector(
-  {
+const github = {
+  name: 'github',
+  connectorId: 'github-actions',
+  config: {
     owner: 'my-org',
     repo: 'my-repo',
-  },
-  {
     token: secret('GITHUB_TOKEN'),
   },
-);
+};
 
 export default defineConfig({
-  connectors: [{ connector: github }],
+  connectors: [github],
   dashboards: {
     engineering: defineDashboard({
       widgets: {
@@ -58,6 +57,10 @@ export default defineConfig({
     }),
   },
 });
+
+// When mounting the engine, register the connector class:
+//   import { GitHubConnector } from '@rawdash/connector-github';
+//   mountEngine(config, { connectorRegistry: { 'github-actions': GitHubConnector } });
 ```
 
 ## API

package/dist/index.d.ts

@@ -1,16 +1,20 @@
-import { RequestObserver, HttpRequest, HttpResponse } from '@rawdash/connector-shared';
+import { RequestObserver, ConnectorLogger, HttpRequest, HttpResponse } from '@rawdash/connector-shared';
+export { ConnectorLogger, ConnectorLoggerOptions, LogFields, createDefaultConnectorLogger, noopConnectorLogger } from '@rawdash/connector-shared';
 import { z } from 'zod';
 
 type Secret = {
     $secret: string;
 };
+type SecretRef = Secret;
 declare function secret(name: string): Secret;
 declare function isSecret(value: unknown): value is Secret;
+declare const secretRefSchema: z.ZodType<SecretRef>;
+declare function withSecretRef<T extends z.ZodTypeAny>(schema: T): z.ZodUnion<[T, z.ZodType<SecretRef>]>;
 interface SecretsResolver {
-    resolve(name: string): string | undefined;
+    resolve(name: string): unknown;
 }
 declare class EnvSecretsResolver implements SecretsResolver {
-    resolve(name: string): string | undefined;
+    resolve(name: string): unknown;
 }
 declare function extractSecretNames(value: unknown): string[];
 declare function resolveSecrets<T>(obj: T, resolver: SecretsResolver): T;
@@ -127,6 +131,13 @@ interface StorageHandle {
     deleteOlderThan(shape: 'events' | 'metrics' | 'distributions', tsUnixMs: number): Promise<{
         rowsDeleted: number;
     }>;
+    getHealth?(): Promise<ConnectorHealth | null>;
+}
+interface ConnectorHealth {
+    status: 'idle' | 'syncing' | 'error' | 'auth_failed' | 'paused';
+    lastSyncAt: string | null;
+    lastError: string | null;
+    syncIntervalSeconds: number;
 }
 interface CredentialField {
     description: string;
@@ -147,6 +158,7 @@ interface SyncOptions {
     mode: 'full' | 'latest';
     since?: string;
     cursor?: unknown;
+    resources?: ReadonlySet<string>;
 }
 interface SyncResult {
     done: boolean;
@@ -162,6 +174,7 @@ interface Connector {
 interface ConnectorContext {
     observer?: RequestObserver;
     secretsResolver?: SecretsResolver;
+    logger?: ConnectorLogger;
 }
 interface ConnectorRequestOptions {
     resource: string;
@@ -180,7 +193,9 @@ declare abstract class BaseConnector<TSettings = unknown, TCreds extends Credent
     protected creds: InferCredentials<TCreds>;
     private rawCredInput;
     private ctx;
+    private cachedLogger;
     constructor(settings: TSettings, creds?: InferCredentialInput<TCreds>, ctx?: ConnectorContext);
+    protected get logger(): ConnectorLogger;
     protected request<T = unknown>(req: HttpRequest, opts: ConnectorRequestOptions): Promise<HttpResponse<T>>;
     protected get<T = unknown>(url: string, opts: ConnectorRequestOptions & {
         headers?: Record<string, string>;
@@ -193,6 +208,7 @@ declare abstract class BaseConnector<TSettings = unknown, TCreds extends Credent
         signal?: AbortSignal;
         rateLimit?: HttpRequest['rateLimit'];
     }): Promise<HttpResponse<T>>;
+    protected isResourceEnabled<R extends string>(resource: R): boolean;
     serializeConfig(): Record<string, unknown>;
     protected sleep(ms: number, signal?: AbortSignal): Promise<void>;
     protected withRetry<T>(fn: (signal?: AbortSignal) => Promise<{
@@ -220,6 +236,8 @@ interface ChunkedSyncCursor<TPhase extends string, TPage> {
     phase: TPhase;
     page: TPage | null;
 }
+declare function selectActivePhases<R extends string, P extends string>(resourceToPhase: (resource: R) => P, order: readonly P[], enabled: readonly R[] | undefined): P[];
+declare function makeChunkedCursorGuard<TPhase extends string>(phases: readonly TPhase[]): (value: unknown) => value is ChunkedSyncCursor<TPhase, string>;
 interface FetchPageResult<TPage> {
     items: unknown[];
     next: TPage | null;
@@ -230,9 +248,20 @@ interface ChunkedSyncOptions<TPhase extends string, TPage> {
     signal: AbortSignal | undefined;
     fetchPage: (phase: TPhase, page: TPage | null, signal: AbortSignal | undefined) => Promise<FetchPageResult<TPage>>;
     writeBatch: (phase: TPhase, items: unknown[], page: TPage | null) => Promise<void>;
+    logger?: ConnectorLogger;
 }
 declare function paginateChunked<TPhase extends string, TPage>(opts: ChunkedSyncOptions<TPhase, TPage>): Promise<SyncResult>;
 
+type FilterOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains';
+interface FilterCondition {
+    field: string;
+    op: FilterOperator;
+    value: string | number | boolean;
+}
+type FilterClause = FilterCondition | {
+    or: FilterCondition[];
+};
+
 interface RetentionConfig {
     maxAge?: number;
     maxSize?: number;
@@ -1204,22 +1233,13 @@ declare function getWidgetSchema(kind: WidgetKind): z.ZodObject<{
 
 type AggFn = 'count' | 'sum' | 'avg' | 'min' | 'max' | 'latest' | 'first';
 type Shape = 'event' | 'entity' | 'metric' | 'edge' | 'distribution';
-type FilterOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains';
-interface FilterCondition {
-    field: string;
-    op: FilterOperator;
-    value: string | number | boolean;
-}
-type FilterClause = FilterCondition | {
-    or: FilterCondition[];
-};
 interface GroupBy {
     field: string;
     granularity: 'hour' | 'day' | 'week' | 'month';
 }
 interface Metric {
     connector: {
-        id: string;
+        name: string;
     };
     shape: Shape;
     name?: string;
@@ -1269,7 +1289,12 @@ interface DistributionWidget {
 type Widget = StatWidget | StatusWidget | TimeseriesWidget | DistributionWidget;
 
 interface ConfiguredConnector {
-    connector: Connector;
+    name: string;
+    connectorId: string;
+    config: Record<string, unknown>;
+    syncIntervalSeconds?: number;
+    enabled?: boolean;
+    displayName?: string;
 }
 interface Dashboard {
     widgets: Record<string, Widget>;
@@ -1296,13 +1321,14 @@ interface SyncState {
 declare const ACTIVE_SYNC_STATUSES: ReadonlySet<SyncStatus>;
 declare function isSyncActive(status: SyncStatus): boolean;
 
-type WidgetSyncState = 'synced' | 'unsynced' | 'syncing' | 'stale' | 'error';
+type WidgetSyncState = 'fresh' | 'stale' | 'unsynced' | 'syncing' | 'failing';
 interface CachedWidget<TData = unknown> {
     widgetId: string;
     connectorId: string;
     data: TData | null;
     cachedAt: string | null;
     syncState?: WidgetSyncState;
+    syncIntervalSeconds?: number;
     meta?: Record<string, unknown>;
 }
 interface WidgetsListResponse {
@@ -1333,8 +1359,162 @@ interface ServerDataSource {
 type ConfigFieldsSchema = z.ZodObject<z.ZodRawShape>;
 declare function defineConfigFields<T extends z.ZodRawShape>(schema: z.ZodObject<T>): z.ZodObject<T>;
 
+declare const connectorCategorySchema: z.ZodEnum<{
+    engineering: "engineering";
+    product: "product";
+    analytics: "analytics";
+    marketing: "marketing";
+    sales: "sales";
+    support: "support";
+    finance: "finance";
+    infrastructure: "infrastructure";
+    security: "security";
+}>;
+type ConnectorCategory = z.infer<typeof connectorCategorySchema>;
+declare const connectorDocSchema: z.ZodObject<{
+    displayName: z.ZodString;
+    category: z.ZodEnum<{
+        engineering: "engineering";
+        product: "product";
+        analytics: "analytics";
+        marketing: "marketing";
+        sales: "sales";
+        support: "support";
+        finance: "finance";
+        infrastructure: "infrastructure";
+        security: "security";
+    }>;
+    tagline: z.ZodString;
+    brandColor: z.ZodOptional<z.ZodString>;
+    vendor: z.ZodObject<{
+        name: z.ZodString;
+        apiDocs: z.ZodOptional<z.ZodURL>;
+        website: z.ZodOptional<z.ZodURL>;
+    }, z.core.$strip>;
+    auth: z.ZodObject<{
+        summary: z.ZodString;
+        setup: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>;
+    rateLimit: z.ZodOptional<z.ZodString>;
+    limitations: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+type ConnectorDoc = z.infer<typeof connectorDocSchema>;
+/**
+ * Declare a connector's documentation metadata. Validates the shape at module
+ * load so a malformed `doc` fails fast (and fails the package's tests), and
+ * returns the typed object for the generator to consume.
+ */
+declare function defineConnectorDoc(doc: ConnectorDoc): ConnectorDoc;
+
+/**
+ * Optional cost signal a connector reports about syncing. Surfaced in the
+ * generated docs as a callout and read by the cloud connector-manager to warn
+ * when a chosen sync interval is more aggressive than recommended. Use it for
+ * connectors where frequent syncs are genuinely expensive (per-query billing,
+ * tight quotas).
+ */
+interface ConnectorCost {
+    /** Recommended sync interval, e.g. "1 day". */
+    recommendedInterval?: string;
+    /** Minimum sensible sync interval, e.g. "1 hour". */
+    minInterval?: string;
+    /** What a single sync costs upstream, e.g. "2 Cost Explorer queries (~$0.02)". */
+    perSync?: string;
+    /** One-line warning shown next to the frequency control / at the top of docs. */
+    warning?: string;
+}
+/**
+ * Map of resource name → Zod schema describing the raw API response shape
+ * for that resource. Resource names must match the `resource` tag passed to
+ * `request()` (see {@link BaseConnector.request}) so the shape-drift pipeline
+ * can correlate observations with their declared shape.
+ *
+ * Consumed by:
+ * - the cloud baseline generator, which walks this map at deploy time to
+ *   populate `connector_baselines`
+ * - property tests in `@rawdash/connector-test-utils`, which fuzz against
+ *   each schema
+ *
+ * See `docs/authoring-a-connector.md` for the authoring guide.
+ */
+type ConnectorSchemas = Readonly<Record<string, z.ZodType>>;
+/**
+ * Compile-time contract every connector class must satisfy. Declaring
+ * `static schemas` is mandatory — without it, the connector cannot be added
+ * to a {@link ConnectorRegistry} and TypeScript will fail the build.
+ */
+type ConnectorClass = {
+    new (settings: never, creds?: never, ctx?: ConnectorContext): Connector;
+    readonly credentials?: CredentialsSchema;
+    readonly schemas: ConnectorSchemas;
+    /** Per-resource definitions (shape, docs, and response schemas). */
+    readonly resources?: ResourceDefinitions;
+    /** Optional cost / recommended-frequency signal (see {@link ConnectorCost}). */
+    readonly cost?: ConnectorCost;
+};
+type ConnectorRegistry = Record<string, ConnectorClass>;
+declare function instantiateConnector(entry: ConfiguredConnector, registry: ConnectorRegistry, secretsResolver?: SecretsResolver, logger?: ConnectorLogger): Connector;
+
+interface ResourceField {
+    name: string;
+    description: string;
+}
+interface ResourceDefBase {
+    description: string;
+    endpoint?: string;
+    notes?: string;
+    dynamic?: boolean;
+    responses?: Readonly<Record<string, z.ZodType>>;
+}
+type ResourceDefinition = (ResourceDefBase & {
+    shape: 'entity';
+    fields?: ResourceField[];
+}) | (ResourceDefBase & {
+    shape: 'event';
+    fields?: ResourceField[];
+}) | (ResourceDefBase & {
+    shape: 'metric';
+    unit?: string;
+    granularity?: string;
+    dimensions?: ResourceField[];
+}) | (ResourceDefBase & {
+    shape: 'distribution';
+    kind?: 'buckets' | 'quantiles';
+    unit?: string;
+}) | (ResourceDefBase & {
+    shape: 'edge';
+    from?: string;
+    to?: string;
+});
+type ResourceDefinitions = Readonly<Record<string, ResourceDefinition>>;
+/**
+ * Validate and return a connector's resource definitions. Call it once at
+ * module scope and reference the result from both `static resources` and
+ * `static schemas` on the connector class.
+ */
+declare function defineResources<const T extends ResourceDefinitions>(defs: T): T;
+type UnionToIntersection<U> = (U extends unknown ? (x: U) => void : never) extends (x: infer I) => void ? I : never;
+type ResponsesOf<T extends ResourceDefinitions> = UnionToIntersection<{
+    [K in keyof T]: T[K] extends {
+        responses: infer R;
+    } ? R extends Readonly<Record<string, z.ZodType>> ? R : object : object;
+}[keyof T]>;
+/**
+ * Flatten every resource definition's `responses` into the flat
+ * `{ responseTag -> Zod schema }` map that `ConnectorClass.schemas` exposes for
+ * the shape-drift baseline generator and property tests. The return type keeps
+ * the exact per-tag schema types, so `z.infer<typeof schemas.<tag>>` works.
+ */
+declare function schemasFromResources<const T extends ResourceDefinitions>(defs: T): ResponsesOf<T> & ConnectorSchemas;
+
 declare function computeMetric(storage: StorageHandle, metric: ComputedMetric): Promise<unknown>;
 
+interface ResourceBackfill {
+    requiredWindowMs: number | undefined;
+}
+type ConnectorBackfill = Map<string, ResourceBackfill>;
+declare function computeConnectorBackfill(config: DashboardConfig): Map<string, ConnectorBackfill>;
+
 interface GetStorageHandleOptions {
     signal?: AbortSignal;
 }
@@ -1342,12 +1522,27 @@ interface ServerStorage {
     getStorageHandle(connectorId: string, options?: GetStorageHandleOptions): StorageHandle;
     getSyncState(): Promise<SyncState>;
     markSyncQueued(): Promise<boolean>;
-    markSyncRunning(): Promise<boolean>;
+    markSyncRunning?(): Promise<boolean>;
     markSyncSucceeded(): Promise<void>;
     markSyncFailed(error: string): Promise<void>;
 }
 
-declare function resolveWidget(widgetId: string, widget: Widget, connectors: ConfiguredConnector[] | readonly string[] | undefined, storage: ServerStorage): Promise<CachedWidget | undefined>;
+declare function resolveWidget(dashboardId: string, widgetId: string, widget: Widget, connectors: readonly string[] | undefined, storage: ServerStorage): Promise<CachedWidget | undefined>;
+
+/**
+ * Stable 32-bit hex hash of a widget's declarative config. Used as the
+ * config-dependent component of the widget ETag so that a config edit
+ * invalidates the cached ETag even when `lastSyncAt` is unchanged.
+ *
+ * Not cryptographic — collision-resistant enough for cache busting.
+ */
+declare function hashWidgetConfig(widget: Widget): string;
+/**
+ * Build the per-widget ETag value (unquoted). Combines the widget's
+ * `lastSyncAt` (the connector's last successful sync timestamp, which is
+ * what `CachedWidget.cachedAt` reflects) and a hash of the widget config.
+ */
+declare function computeWidgetEtag(lastSyncAt: string | null, widget: Widget): string;
 
 declare function withAbortSignal(handle: StorageHandle, signal: AbortSignal): StorageHandle;
 
@@ -1357,6 +1552,7 @@ declare class InMemoryStorage implements ServerStorage {
     private metricStore;
     private edgeStore;
     private distributionStore;
+    private lastWriteAt;
     private syncState;
     getStorageHandle(connectorId: string, options?: GetStorageHandleOptions): StorageHandle;
     private buildHandle;
@@ -1402,4 +1598,4 @@ type WireDashboard = z.infer<typeof wireDashboardSchema>;
 type WireConfig = z.infer<typeof wireConfigSchema>;
 declare function toWireConfig(config: DashboardConfig): WireConfig;
 
-export { ACTIVE_SYNC_STATUSES, type AggFn, BaseConnector, type CachedWidget, type ChunkedSyncCursor, type ChunkedSyncOptions, type ComputedMetric, type ConfigFieldsSchema, type ConfiguredConnector, type Connector, type ConnectorContext, type ConnectorRequestOptions, type CredentialField, type CredentialsSchema, type Dashboard, type DashboardConfig, type DataSource, type Distribution, type DistributionQuery, type DistributionWidget, type Edge, type EdgeQuery, type Entity, type EntityQuery, EnvSecretsResolver, type Event, type EventQuery, type FetchPageResult, type FilterClause, type FilterCondition, type FilterOperator, type GetStorageHandleOptions, type GroupBy, type HealthResponse, InMemoryStorage, type InferCredentialInput, type InferCredentials, type JSONValue, type Metric, type MetricQuery, type MetricSample, type RetentionConfig, type RetentionDeletionPlan, type Secret, type SecretsResolver, type ServerDataSource, type ServerStorage, type Shape, type StatWidget, type StatusWidget, type StorageHandle, type SyncOptions, type SyncResult, type SyncState, type SyncStatus, type TimeseriesWidget, type TriggerSyncResponse, type Widget, type WidgetKind, type WidgetSyncState, type WidgetsListResponse, type WireConfig, type WireConnector, type WireDashboard, aggFnSchema, computeMetric, computeRetention, computedMetricSchema, defineConfig, defineConfigFields, defineConnector, defineDashboard, defineMetric, distributionWidgetSchema, extractSecretNames, filterClauseSchema, filterConditionSchema, filterOperatorSchema, getWidgetSchema, groupBySchema, isSecret, isSyncActive, paginateChunked, resolveSecrets, resolveWidget, secret, selectForDeletion, shapeSchema, statWidgetSchema, statusWidgetSchema, timeseriesWidgetSchema, toWireConfig, widgetSchema, widgetSchemas, wireConfigSchema, wireConnectorSchema, wireDashboardSchema, withAbortSignal };
+export { ACTIVE_SYNC_STATUSES, type AggFn, BaseConnector, type CachedWidget, type ChunkedSyncCursor, type ChunkedSyncOptions, type ComputedMetric, type ConfigFieldsSchema, type ConfiguredConnector, type Connector, type ConnectorBackfill, type ConnectorCategory, type ConnectorClass, type ConnectorContext, type ConnectorCost, type ConnectorDoc, type ConnectorHealth, type ConnectorRegistry, type ConnectorRequestOptions, type ConnectorSchemas, type CredentialField, type CredentialsSchema, type Dashboard, type DashboardConfig, type DataSource, type Distribution, type DistributionQuery, type DistributionWidget, type Edge, type EdgeQuery, type Entity, type EntityQuery, EnvSecretsResolver, type Event, type EventQuery, type FetchPageResult, type FilterClause, type FilterCondition, type FilterOperator, type GetStorageHandleOptions, type GroupBy, type HealthResponse, InMemoryStorage, type InferCredentialInput, type InferCredentials, type JSONValue, type Metric, type MetricQuery, type MetricSample, type ResourceBackfill, type ResourceDefinition, type ResourceDefinitions, type ResourceField, type RetentionConfig, type RetentionDeletionPlan, type Secret, type SecretRef, type SecretsResolver, type ServerDataSource, type ServerStorage, type Shape, type StatWidget, type StatusWidget, type StorageHandle, type SyncOptions, type SyncResult, type SyncState, type SyncStatus, type TimeseriesWidget, type TriggerSyncResponse, type Widget, type WidgetKind, type WidgetSyncState, type WidgetsListResponse, type WireConfig, type WireConnector, type WireDashboard, aggFnSchema, computeConnectorBackfill, computeMetric, computeRetention, computeWidgetEtag, computedMetricSchema, connectorCategorySchema, connectorDocSchema, defineConfig, defineConfigFields, defineConnector, defineConnectorDoc, defineDashboard, defineMetric, defineResources, distributionWidgetSchema, extractSecretNames, filterClauseSchema, filterConditionSchema, filterOperatorSchema, getWidgetSchema, groupBySchema, hashWidgetConfig, instantiateConnector, isSecret, isSyncActive, makeChunkedCursorGuard, paginateChunked, resolveSecrets, resolveWidget, schemasFromResources, secret, secretRefSchema, selectActivePhases, selectForDeletion, shapeSchema, statWidgetSchema, statusWidgetSchema, timeseriesWidgetSchema, toWireConfig, widgetSchema, widgetSchemas, wireConfigSchema, wireConnectorSchema, wireDashboardSchema, withAbortSignal, withSecretRef };