@dbx-tools/appkit-generator 0.0.0-stage → 0.9.138

package/src/config.ts

@@ -0,0 +1,98 @@
+/**
+ * AppKit generator-plugin configuration and environment resolution.
+ *
+ * @module
+ */
+
+import type { BasePluginConfig } from "@databricks/appkit";
+import { configUtils } from "@dbx-tools/core";
+import { stringUtils } from "@dbx-tools/shared-core";
+import manifest from "./manifest.json" with { type: "json" };
+import {
+  DEFAULT_MAX_SOURCES,
+  DEFAULT_SOURCE_MAX_LENGTH,
+  MAX_SOURCES,
+  MAX_SOURCE_MAX_LENGTH,
+} from "./research.ts";
+
+/** Default ceiling for one complete generator operation. */
+export const DEFAULT_GENERATOR_TIMEOUT_MS = 240_000;
+
+/** AppKit configuration accepted by the generator plugin. */
+export interface GeneratorPluginConfig extends BasePluginConfig {
+  /** Model endpoint or loose model name used for structured synthesis. */
+  model?: string;
+  /** Maximum cited pages fetched per research operation. */
+  maxSources?: number;
+  /** Maximum readable characters retained from each fetched page. */
+  sourceMaxLength?: number;
+  /** Overall operation timeout in milliseconds. */
+  timeoutMs?: number;
+  /** Unity Catalog containing all generated per-user schemas. */
+  catalog?: string;
+  /** SQL warehouse used for data generation and validation. */
+  warehouseId?: string;
+}
+
+/** Fully resolved generator configuration. */
+export interface ResolvedGeneratorConfig {
+  /** Optional pinned synthesis model. */
+  model?: string;
+  /** Bounded cited-page count. */
+  maxSources: number;
+  /** Character cap per fetched source. */
+  sourceMaxLength: number;
+  /** Overall operation timeout in milliseconds. */
+  timeoutMs: number;
+  /** Unity Catalog containing generated schemas, when configured. */
+  catalog?: string;
+  /** SQL warehouse used for mutations, when configured. */
+  warehouseId?: string;
+}
+
+/** JSON Schema published by the generator plugin manifest. */
+export const GENERATOR_CONFIG_SCHEMA = manifest.config.schema;
+
+/** Resolve explicit plugin values over environment values and defaults. */
+export function resolveGeneratorConfig(
+  config: GeneratorPluginConfig = {},
+): ResolvedGeneratorConfig {
+  const model =
+    stringUtils.trimToNull(config.model) ??
+    configUtils.text("GENERATOR_MODEL", configUtils.ENV_ONLY);
+  const catalog =
+    stringUtils.trimToNull(config.catalog) ??
+    configUtils.text("GENERATOR_CATALOG", configUtils.ENV_ONLY);
+  const warehouseId =
+    stringUtils.trimToNull(config.warehouseId) ??
+    configUtils.text("DATABRICKS_WAREHOUSE_ID", configUtils.ENV_ONLY);
+  return {
+    ...(model ? { model } : {}),
+    ...(catalog ? { catalog } : {}),
+    ...(warehouseId ? { warehouseId } : {}),
+    maxSources: Math.min(
+      configUtils.positiveInt(
+        config.maxSources,
+        "GENERATOR_MAX_SOURCES",
+        DEFAULT_MAX_SOURCES,
+        configUtils.ENV_ONLY,
+      ),
+      MAX_SOURCES,
+    ),
+    sourceMaxLength: Math.min(
+      configUtils.positiveInt(
+        config.sourceMaxLength,
+        "GENERATOR_SOURCE_MAX_LENGTH",
+        DEFAULT_SOURCE_MAX_LENGTH,
+        configUtils.ENV_ONLY,
+      ),
+      MAX_SOURCE_MAX_LENGTH,
+    ),
+    timeoutMs: configUtils.positiveInt(
+      config.timeoutMs,
+      "GENERATOR_TIMEOUT_MS",
+      DEFAULT_GENERATOR_TIMEOUT_MS,
+      configUtils.ENV_ONLY,
+    ),
+  };
+}

package/src/dashboard.ts

@@ -0,0 +1,295 @@
+/**
+ * Deterministic AI/BI dashboard assembly and OBO publication.
+ *
+ * @module
+ */
+
+import { ExecutionError } from "@databricks/appkit";
+import { databricks } from "@dbx-tools/appkit";
+import { hash } from "@dbx-tools/shared-core";
+import type { BrandContext } from "@dbx-tools/shared-core/brand-utils";
+import {
+  GeneratedDashboardSchema,
+  type GeneratedDashboard,
+  type GeneratedMetric,
+  type GeneratedSchema,
+  type GeneratorPlan,
+} from "@dbx-tools/shared-generator/workflow";
+import { z } from "zod";
+import { resolveGeneratorIdentity } from "./namespace.ts";
+
+const SerializedDashboardSchema = z
+  .object({
+    datasets: z.array(z.record(z.string(), z.unknown())).min(1),
+    pages: z.array(z.record(z.string(), z.unknown())).min(1),
+    uiSettings: z.record(z.string(), z.unknown()),
+  })
+  .strict();
+
+/** Inputs for dashboard assembly and publication. */
+export interface CreateGeneratorDashboardInput {
+  /** Generated schema containing the fact table. */
+  location: GeneratedSchema;
+  /** Validated data and metric plan. */
+  plan: GeneratorPlan;
+  /** Brand applied to the dashboard theme. */
+  brand: BrandContext;
+  /** SQL warehouse used by the dashboard. */
+  warehouseId: string;
+  /** Company display name. */
+  company: string;
+  /** Demo purpose. */
+  purpose: string;
+}
+
+/** Compile one metric into a reusable Lakeview dataset measure. */
+function measure(metric: GeneratedMetric) {
+  const expression =
+    metric.aggregation === "count"
+      ? "COUNT(`*`)"
+      : `${metric.aggregation === "sum" ? "SUM" : "AVG"}(\`${metric.column}\`)`;
+  return {
+    displayName: metric.label,
+    description: metric.definition,
+    expression,
+  };
+}
+
+/** Counter widget bound to one dataset measure. */
+function counterWidget(metric: GeneratedMetric, index: number) {
+  const fieldName = `measure(${metric.label})`;
+  return {
+    widget: {
+      name: `kpi_${metric.name}`,
+      queries: [
+        {
+          name: "main_query",
+          query: {
+            datasetName: "generated_facts",
+            fields: [{ name: fieldName, expression: `MEASURE(\`${metric.label}\`)` }],
+            disaggregated: false,
+          },
+        },
+      ],
+      spec: {
+        version: 2,
+        widgetType: "counter",
+        encodings: {
+          value: {
+            fieldName,
+            displayName: metric.label,
+            ...(metric.unit === "percent"
+              ? {
+                  format: {
+                    type: "number-percent",
+                    decimalPlaces: { type: "max", places: 2 },
+                  },
+                }
+              : {}),
+          },
+        },
+        frame: { showTitle: true, title: metric.label },
+      },
+    },
+    position: {
+      x: (index % 3) * 4,
+      y: 2 + Math.floor(index / 3) * 3,
+      width: 4,
+      height: 3,
+    },
+  };
+}
+
+/** Build one complete dashboard with automatic companion Genie enabled. */
+export function buildGeneratorDashboard(input: CreateGeneratorDashboardInput) {
+  const fullTable = `${input.location.catalog}.${input.location.schema}.${input.plan.table}`;
+  const dateColumn = input.plan.columns.find(({ generator }) => generator === "date")?.name;
+  const categoryColumn = input.plan.columns.find(({ generator }) => generator === "category")?.name;
+  const metric = input.plan.metrics[0]!;
+  const metricField = `measure(${metric.label})`;
+  const kpiRows = Math.ceil(input.plan.metrics.length / 3);
+  const chartY = 2 + kpiRows * 3;
+  const layout: Record<string, unknown>[] = [
+    {
+      widget: {
+        name: "header",
+        multilineTextboxSpec: {
+          lines: [`# ${input.company}: ${input.purpose}\n`, `\n`, input.plan.story],
+        },
+      },
+      position: { x: 0, y: 0, width: 12, height: 2 },
+    },
+    ...input.plan.metrics.map(counterWidget),
+  ];
+  if (dateColumn) {
+    layout.push({
+      widget: {
+        name: "metric_trend",
+        queries: [
+          {
+            name: "main_query",
+            query: {
+              datasetName: "generated_facts",
+              fields: [
+                {
+                  name: `daily(${dateColumn})`,
+                  expression: `DATE_TRUNC("DAY", \`${dateColumn}\`)`,
+                },
+                { name: metricField, expression: `MEASURE(\`${metric.label}\`)` },
+              ],
+              disaggregated: false,
+            },
+          },
+        ],
+        spec: {
+          version: 3,
+          widgetType: "line",
+          encodings: {
+            x: {
+              fieldName: `daily(${dateColumn})`,
+              scale: { type: "temporal" },
+              displayName: "Date",
+            },
+            y: {
+              fieldName: metricField,
+              scale: { type: "quantitative" },
+              displayName: metric.label,
+            },
+          },
+          frame: { showTitle: true, title: `${metric.label} trend` },
+        },
+      },
+      position: { x: 0, y: chartY, width: 7, height: 6 },
+    });
+  }
+  if (categoryColumn) {
+    layout.push({
+      widget: {
+        name: "metric_by_category",
+        queries: [
+          {
+            name: "main_query",
+            query: {
+              datasetName: "generated_facts",
+              fields: [
+                { name: categoryColumn, expression: `\`${categoryColumn}\`` },
+                { name: metricField, expression: `MEASURE(\`${metric.label}\`)` },
+              ],
+              disaggregated: false,
+            },
+          },
+        ],
+        spec: {
+          version: 3,
+          widgetType: "bar",
+          encodings: {
+            x: {
+              fieldName: categoryColumn,
+              scale: { type: "categorical" },
+              displayName: categoryColumn,
+            },
+            y: {
+              fieldName: metricField,
+              scale: { type: "quantitative" },
+              displayName: metric.label,
+            },
+          },
+          frame: { showTitle: true, title: `${metric.label} by ${categoryColumn}` },
+        },
+      },
+      position: { x: 7, y: chartY, width: 5, height: 6 },
+    });
+  }
+  return SerializedDashboardSchema.parse({
+    datasets: [
+      {
+        name: "generated_facts",
+        displayName: input.plan.table,
+        asset_name: fullTable,
+        columns: input.plan.metrics.map(measure),
+      },
+    ],
+    pages: [
+      {
+        name: "overview",
+        displayName: "Overview",
+        pageType: "PAGE_TYPE_CANVAS",
+        layoutVersion: "GRID_V1",
+        layout,
+      },
+    ],
+    uiSettings: {
+      theme: {
+        canvasBackgroundColor: {
+          light: input.brand.colors.background,
+          dark: input.brand.colors.foreground,
+        },
+        widgetBackgroundColor: {
+          light: input.brand.colors.surface,
+          dark: input.brand.colors.primary,
+        },
+        fontColor: {
+          light: input.brand.colors.foreground,
+          dark: input.brand.colors.background,
+        },
+        selectionColor: {
+          light: input.brand.colors.primary,
+          dark: input.brand.colors.accent,
+        },
+        visualizationColors: [
+          input.brand.colors.primary,
+          input.brand.colors.accent,
+          input.brand.colors.primaryHover,
+          input.brand.colors.muted,
+        ],
+        widgetHeaderAlignment: "LEFT",
+      },
+      genieSpace: {
+        isEnabled: true,
+        enablementMode: "AUTO",
+      },
+    },
+  });
+}
+
+/** Create and publish a dashboard as the active OBO user. */
+export async function createGeneratorDashboard(
+  input: CreateGeneratorDashboardInput,
+  signal?: AbortSignal,
+): Promise<GeneratedDashboard> {
+  const identity = await resolveGeneratorIdentity();
+  if (identity.principal !== input.location.principal) {
+    throw new ExecutionError("Dashboard principal does not match the active OBO user");
+  }
+  const legacy = identity.userClient.toLegacyWorkspaceClient();
+  const parentPath = `/Workspace/Users/${identity.principal}/.dbx-tools/generator/${input.location.schema}`;
+  const context = signal ? databricks.toContext(signal) : undefined;
+  await legacy.workspace.mkdirs({ path: parentPath }, context);
+  const created = await legacy.lakeview.create(
+    {
+      dashboard: {
+        display_name: `${input.company} ${input.purpose} ${hash.id(6)}`,
+        parent_path: parentPath,
+        warehouse_id: input.warehouseId,
+        serialized_dashboard: JSON.stringify(buildGeneratorDashboard(input)),
+      },
+    },
+    context,
+  );
+  if (!created.dashboard_id) {
+    throw new ExecutionError("Lakeview create returned no dashboard id");
+  }
+  await legacy.lakeview.publish(
+    {
+      dashboard_id: created.dashboard_id,
+      warehouse_id: input.warehouseId,
+      embed_credentials: false,
+    },
+    context,
+  );
+  return GeneratedDashboardSchema.parse({
+    dashboardId: created.dashboard_id,
+    ...(created.path ? { path: created.path } : {}),
+    companionGenie: "AUTO",
+  });
+}

package/src/data.ts

@@ -0,0 +1,167 @@
+/**
+ * Deterministic Databricks SQL compilation, execution, and validation.
+ *
+ * @module
+ */
+
+import { ExecutionError, type WorkspaceClient } from "@databricks/appkit";
+import { databricks } from "@dbx-tools/appkit";
+import { asyncUtils } from "@dbx-tools/shared-core";
+import { MaterializedGeneratorDataSchema } from "@dbx-tools/shared-generator/workflow";
+import type {
+  GeneratedColumn,
+  GeneratedMetric,
+  GeneratedSchema,
+  GeneratorPlan,
+  MaterializedGeneratorData,
+} from "@dbx-tools/shared-generator/workflow";
+import { resolveGeneratorIdentity } from "./namespace.ts";
+
+const TERMINAL_STATEMENT_STATES = new Set(["SUCCEEDED", "FAILED", "CANCELED", "CLOSED"]);
+type StatementResponse = Awaited<ReturnType<WorkspaceClient["statementExecution"]["getStatement"]>>;
+
+/** Quote one already-validated SQL identifier. */
+function quoteIdentifier(value: string): string {
+  return `\`${value.replaceAll("`", "``")}\``;
+}
+
+/** Quote one SQL string literal. */
+function quoteLiteral(value: string): string {
+  return `'${value.replaceAll("'", "''")}'`;
+}
+
+/** Compile one safe column generator into a SQL expression. */
+function columnExpression(column: GeneratedColumn): string {
+  const salt = quoteLiteral(column.name);
+  switch (column.generator) {
+    case "id":
+      return `concat('ID-', lpad(cast(id as string), 8, '0'))`;
+    case "date":
+      return `date_sub(current_date(), pmod(xxhash64(id, ${salt}), ${column.days}))`;
+    case "category": {
+      const values = column.values.map(quoteLiteral);
+      const fallback = `element_at(array(${values.join(", ")}), pmod(xxhash64(id, ${salt}), ${values.length}) + 1)`;
+      return `case when pmod(xxhash64(id, ${salt}), 100) < 55 then ${values[0]} when pmod(xxhash64(id, ${salt}), 100) < 80 then ${values[1]} else ${fallback} end`;
+    }
+    case "number": {
+      const spread = column.maximum - column.minimum;
+      return `round((${column.minimum} + (pmod(xxhash64(id, ${salt}), 1000000) / 999999.0) * ${spread}) * case when pmod(xxhash64(id, ${salt}, 'anomaly'), 100) < 8 then 1.65 else 1 end, ${column.decimals})`;
+    }
+    case "boolean":
+      return `pmod(xxhash64(id, ${salt}), 10000) < ${Math.round(column.trueProbability * 10_000)}`;
+    default: {
+      const exhaustive: never = column;
+      throw new TypeError(`Unsupported generated column: ${String(exhaustive)}`);
+    }
+  }
+}
+
+/** Compile the generated table statement without accepting model-authored SQL. */
+export function compileGeneratorTableSql(location: GeneratedSchema, plan: GeneratorPlan): string {
+  const columns = plan.columns.map(
+    (column) => `${columnExpression(column)} AS ${quoteIdentifier(column.name)}`,
+  );
+  return [
+    `CREATE OR REPLACE TABLE ${quoteIdentifier(location.catalog)}.${quoteIdentifier(location.schema)}.${quoteIdentifier(plan.table)}`,
+    `COMMENT ${quoteLiteral(plan.story)}`,
+    "AS SELECT",
+    `  ${columns.join(",\n  ")}`,
+    `FROM range(${plan.rowCount})`,
+  ].join("\n");
+}
+
+/** Compile one KPI validation query from its typed aggregate. */
+export function compileMetricValidationSql(table: string, metric: GeneratedMetric): string {
+  const aggregate =
+    metric.aggregation === "count"
+      ? "count(*)"
+      : `${metric.aggregation === "sum" ? "sum" : "avg"}(${quoteIdentifier(metric.column!)})`;
+  return `SELECT ${aggregate} AS ${quoteIdentifier(metric.name)} FROM ${table}`;
+}
+
+/** Execute one statement and wait for its terminal response. */
+export async function executeGeneratorStatement(
+  client: WorkspaceClient,
+  warehouseId: string,
+  statement: string,
+  location: Pick<GeneratedSchema, "catalog" | "schema">,
+  signal?: AbortSignal,
+): Promise<StatementResponse> {
+  const context = signal ? databricks.toContext(signal) : undefined;
+  let response = await client.statementExecution.executeStatement(
+    {
+      warehouse_id: warehouseId,
+      statement,
+      catalog: location.catalog,
+      schema: location.schema,
+      wait_timeout: "50s",
+      on_wait_timeout: "CONTINUE",
+    },
+    context,
+  );
+  if (!TERMINAL_STATEMENT_STATES.has(response.status?.state ?? "")) {
+    const statementId = response.statement_id;
+    if (!statementId) throw new ExecutionError("Statement execution returned no statement id");
+    for await (const polled of asyncUtils.poll<StatementResponse>(
+      ({ signal: pollSignal }) =>
+        client.statementExecution.getStatement(
+          { statement_id: statementId },
+          databricks.toContext(pollSignal),
+        ),
+      {
+        intervalMs: 1_000,
+        timeoutMs: 180_000,
+        ...(signal ? { signal } : {}),
+        predicate: (value) => !TERMINAL_STATEMENT_STATES.has(value.status?.state ?? ""),
+      },
+    )) {
+      response = polled;
+    }
+  }
+  if (response.status?.state !== "SUCCEEDED") {
+    throw new ExecutionError(
+      response.status?.error?.message ??
+        `Statement ended in ${response.status?.state ?? "UNKNOWN"}`,
+    );
+  }
+  return response;
+}
+
+/** Materialize generated data as the OBO user and validate every planned KPI. */
+export async function materializeGeneratorData(
+  location: GeneratedSchema,
+  plan: GeneratorPlan,
+  warehouseId: string,
+  signal?: AbortSignal,
+): Promise<MaterializedGeneratorData> {
+  const identity = await resolveGeneratorIdentity();
+  if (identity.principal !== location.principal) {
+    throw new ExecutionError("Generator schema principal does not match the active OBO user");
+  }
+  await executeGeneratorStatement(
+    identity.userClient,
+    warehouseId,
+    compileGeneratorTableSql(location, plan),
+    location,
+    signal,
+  );
+  const fullTable = `${location.catalog}.${location.schema}.${plan.table}`;
+  const validationStatements: Record<string, string> = {};
+  for (const metric of plan.metrics) {
+    const response = await executeGeneratorStatement(
+      identity.userClient,
+      warehouseId,
+      compileMetricValidationSql(fullTable, metric),
+      location,
+      signal,
+    );
+    if (!response.statement_id) {
+      throw new ExecutionError(`Validation for ${metric.name} returned no statement id`);
+    }
+    validationStatements[metric.name] = response.statement_id;
+  }
+  return MaterializedGeneratorDataSchema.parse({
+    tables: [fullTable],
+    validationStatements,
+  });
+}

package/src/defaults.ts

@@ -0,0 +1,61 @@
+/**
+ * AppKit interceptor settings for researched brand generation.
+ *
+ * @module
+ */
+
+import { brandRequestSubject, type BrandGenerationRequest } from "./schema.ts";
+
+/** AppKit execute configuration used by this package. */
+export type BrandExecuteConfig = {
+  cache?: { enabled?: boolean; ttl?: number; cacheKey?: (string | number | object)[] };
+  retry?: { enabled?: boolean; attempts?: number; initialDelay?: number; maxDelay?: number };
+  timeout?: number;
+};
+
+/** Execution settings accepted by `Plugin.execute()`. */
+export type BrandExecutionSettings = {
+  default: BrandExecuteConfig;
+  user?: BrandExecuteConfig;
+};
+
+/** Cache lifetime for a generated branding package. */
+export const BRAND_CACHE_TTL_SECONDS = 21_600;
+
+/** Cache-key version for the generated package shape and asset policy. */
+export const BRAND_CACHE_VERSION = 5;
+
+/** Attempts for the idempotent structured synthesis call. */
+export const BRAND_GENERATION_ATTEMPTS = 2;
+
+/** Build request-specific interceptor settings for brand generation. */
+export function brandGenerationSettings(
+  request: BrandGenerationRequest,
+  options: {
+    timeoutMs: number;
+    model?: string;
+    maxSources: number;
+    sourceMaxLength: number;
+  },
+): BrandExecutionSettings {
+  return {
+    default: {
+      cache: {
+        enabled: true,
+        ttl: BRAND_CACHE_TTL_SECONDS,
+        cacheKey: [
+          "brand",
+          "generate",
+          BRAND_CACHE_VERSION,
+          brandRequestSubject(request).toLowerCase(),
+          request.website ?? "",
+          options.model ?? "",
+          options.maxSources,
+          options.sourceMaxLength,
+        ],
+      },
+      retry: { enabled: true, attempts: BRAND_GENERATION_ATTEMPTS },
+      timeout: options.timeoutMs,
+    },
+  };
+}