p-dev-harness 0.3.1 → 0.4.0

package/workspace-snapshot/files/src/observability/posthog-dashboard-contract.ts

@@ -0,0 +1,277 @@
+import {
+  ANALYTICS_EVENT_NAMES,
+  allowedAnalyticsPropertyKeysForEvent,
+} from "./privacy-schema.js";
+import {
+  GUIDED_DISPLAY_STEP_IDS,
+  guidedDisplayStepNumber,
+} from "./analytics-schemas.js";
+import { OBSERVABILITY_SCHEMA_VERSION } from "./constants.js";
+import type { AnalyticsEvent } from "./types.js";
+
+export const POSTHOG_ONBOARDING_HEALTH_DASHBOARD_NAME =
+  "p-dev Packaged Onboarding Health" as const;
+
+export const POSTHOG_ONBOARDING_HEALTH_DASHBOARD_DESCRIPTION =
+  "Consent-gated packaged product analytics for p-dev-harness onboarding and workspace provisioning. Only installations that affirmatively enable anonymous analytics are visible. Counts must not be interpreted as total installs, and this dashboard must not be used to calculate a consent rate." as const;
+
+export const POSTHOG_ONBOARDING_HEALTH_DEFAULT_DATE_RANGE = "-30d" as const;
+
+export const POSTHOG_DASHBOARD_FILTER_PROPERTIES = [
+  "package_version",
+  "release_sha",
+  "os_family",
+] as const;
+
+export const POSTHOG_DASHBOARD_EXCLUDED_METRICS = [
+  "consent_rate",
+  "analytics_opt_in_rate",
+  "error_reporting_opt_in_rate",
+] as const;
+
+export const POSTHOG_INTERPRETATION_MARKDOWN = `## Interpretation
+
+This dashboard contains only anonymous product analytics from packaged p-dev-harness installations that affirmatively enabled analytics.
+
+Non-consenting installations are intentionally invisible. Do not calculate adoption or consent rates from this data.
+
+Use package_version and release_sha to compare releases. Telemetry is supporting evidence, not an authoritative trigger for releases, security actions, or product decisions.`;
+
+export const CONFIGURE_FUNNEL_SUCCESS_OUTCOMES = [
+  "success",
+  "skipped_already_complete",
+] as const;
+
+export type PostHogDashboardInsightKind =
+  | "text"
+  | "trends"
+  | "funnel"
+  | "breakdown";
+
+export interface PostHogDashboardInsightSpec {
+  name: string;
+  kind: PostHogDashboardInsightKind;
+  events: readonly (typeof ANALYTICS_EVENT_NAMES)[number][];
+  breakdownProperties?: readonly string[];
+  secondaryBreakdownProperties?: readonly string[];
+  display?: "line" | "bar" | "stacked_bar" | "table";
+  funnelStepFilters?: readonly ConfigureFunnelStepFilter[];
+}
+
+export interface ConfigureFunnelStepFilter {
+  stepId: (typeof GUIDED_DISPLAY_STEP_IDS)[number];
+  completionOutcomes: readonly (typeof CONFIGURE_FUNNEL_SUCCESS_OUTCOMES)[number][];
+}
+
+function configureStepFilter(
+  stepId: (typeof GUIDED_DISPLAY_STEP_IDS)[number],
+): ConfigureFunnelStepFilter {
+  return {
+    stepId,
+    completionOutcomes: CONFIGURE_FUNNEL_SUCCESS_OUTCOMES,
+  };
+}
+
+export const CONFIGURE_FUNNEL_STEP_FILTERS: readonly ConfigureFunnelStepFilter[] =
+  GUIDED_DISPLAY_STEP_IDS.map(configureStepFilter);
+
+export const CONFIGURE_FUNNEL_STEPS_1_4: readonly ConfigureFunnelStepFilter[] =
+  CONFIGURE_FUNNEL_STEP_FILTERS.slice(0, 4);
+
+export const CONFIGURE_FUNNEL_STEPS_5_8: readonly ConfigureFunnelStepFilter[] =
+  CONFIGURE_FUNNEL_STEP_FILTERS.slice(4);
+
+export const POSTHOG_ONBOARDING_HEALTH_INSIGHTS: readonly PostHogDashboardInsightSpec[] =
+  [
+    {
+      name: "Interpretation",
+      kind: "text",
+      events: [],
+    },
+    {
+      name: "Packaged Sessions by Release",
+      kind: "trends",
+      events: ["p_dev_session_started"],
+      breakdownProperties: ["package_version"],
+      display: "line",
+    },
+    {
+      name: "Packaged Sessions by OS",
+      kind: "trends",
+      events: ["p_dev_session_started"],
+      breakdownProperties: ["os_family"],
+      display: "bar",
+    },
+    {
+      name: "Session to Setup Completion",
+      kind: "funnel",
+      events: ["p_dev_session_started", "p_dev_setup_completed"],
+      breakdownProperties: ["package_version"],
+    },
+    {
+      name: "Configure Step Completion Funnel",
+      kind: "funnel",
+      events: ["p_dev_configure_step_completed"],
+      funnelStepFilters: CONFIGURE_FUNNEL_STEP_FILTERS,
+    },
+    {
+      name: "Configure Funnel — Steps 1–4",
+      kind: "funnel",
+      events: ["p_dev_configure_step_completed"],
+      funnelStepFilters: CONFIGURE_FUNNEL_STEPS_1_4,
+    },
+    {
+      name: "Configure Funnel — Steps 5–8",
+      kind: "funnel",
+      events: ["p_dev_configure_step_completed"],
+      funnelStepFilters: CONFIGURE_FUNNEL_STEPS_5_8,
+    },
+    {
+      name: "Configure Outcomes by Step",
+      kind: "breakdown",
+      events: ["p_dev_configure_step_completed"],
+      breakdownProperties: ["step_id"],
+      secondaryBreakdownProperties: ["completion_outcome"],
+      display: "stacked_bar",
+    },
+    {
+      name: "Workspace Provisioning Outcomes",
+      kind: "trends",
+      events: [
+        "p_dev_workspace_provision_started",
+        "p_dev_workspace_provision_completed",
+        "p_dev_workspace_provision_failed",
+      ],
+      breakdownProperties: ["package_version"],
+      display: "line",
+    },
+    {
+      name: "Provisioning Failure Categories",
+      kind: "breakdown",
+      events: ["p_dev_workspace_provision_failed"],
+      breakdownProperties: ["failure_category"],
+      display: "bar",
+    },
+    {
+      name: "Provisioning Duration Buckets",
+      kind: "breakdown",
+      events: [
+        "p_dev_workspace_provision_completed",
+        "p_dev_workspace_provision_failed",
+      ],
+      breakdownProperties: ["duration_bucket"],
+      display: "bar",
+    },
+    {
+      name: "Provisioning Retry Buckets",
+      kind: "breakdown",
+      events: [
+        "p_dev_workspace_provision_completed",
+        "p_dev_workspace_provision_failed",
+      ],
+      breakdownProperties: ["retry_count_bucket"],
+      display: "bar",
+    },
+    {
+      name: "Rate-Limit Pause Buckets",
+      kind: "breakdown",
+      events: [
+        "p_dev_workspace_provision_completed",
+        "p_dev_workspace_provision_failed",
+      ],
+      breakdownProperties: ["rate_limit_pause_count_bucket"],
+      display: "bar",
+    },
+  ] as const;
+
+export const POSTHOG_ONBOARDING_HEALTH_REQUIRED_INSIGHT_NAMES =
+  POSTHOG_ONBOARDING_HEALTH_INSIGHTS.filter((insight) => insight.kind !== "text")
+    .filter(
+      (insight) =>
+        insight.name !== "Configure Funnel — Steps 1–4" &&
+        insight.name !== "Configure Funnel — Steps 5–8",
+    )
+    .map((insight) => insight.name);
+
+export const POSTHOG_ONBOARDING_HEALTH_OPTIONAL_SPLIT_FUNNEL_NAMES = [
+  "Configure Funnel — Steps 1–4",
+  "Configure Funnel — Steps 5–8",
+] as const;
+
+export function assertDashboardContractUsesAllowlistedEvents(): void {
+  const allowed = new Set<string>(ANALYTICS_EVENT_NAMES);
+  for (const insight of POSTHOG_ONBOARDING_HEALTH_INSIGHTS) {
+    for (const eventName of insight.events) {
+      if (!allowed.has(eventName)) {
+        throw new Error(`Dashboard insight "${insight.name}" uses unknown event.`);
+      }
+    }
+  }
+}
+
+export function assertDashboardContractUsesAllowlistedProperties(): void {
+  for (const insight of POSTHOG_ONBOARDING_HEALTH_INSIGHTS) {
+    for (const eventName of insight.events) {
+      const event = { type: eventName } as AnalyticsEvent;
+      const allowed = new Set(allowedAnalyticsPropertyKeysForEvent(event));
+      for (const property of [
+        ...(insight.breakdownProperties ?? []),
+        ...(insight.secondaryBreakdownProperties ?? []),
+      ]) {
+        if (!allowed.has(property)) {
+          throw new Error(
+            `Dashboard insight "${insight.name}" uses disallowed property "${property}".`,
+          );
+        }
+      }
+    }
+  }
+}
+
+export function configureFunnelStepNumber(
+  stepId: (typeof GUIDED_DISPLAY_STEP_IDS)[number],
+): number {
+  return guidedDisplayStepNumber(stepId);
+}
+
+export const POSTHOG_DASHBOARD_SCHEMA_VERSION = OBSERVABILITY_SCHEMA_VERSION;
+
+export function configureStepEventProperties(
+  stepId: (typeof GUIDED_DISPLAY_STEP_IDS)[number],
+): Array<{
+  key: string;
+  type: "event";
+  operator: "exact";
+  value: string[];
+}> {
+  return [
+    {
+      key: "step_id",
+      type: "event",
+      operator: "exact",
+      value: [stepId],
+    },
+    {
+      key: "completion_outcome",
+      type: "event",
+      operator: "exact",
+      value: [...CONFIGURE_FUNNEL_SUCCESS_OUTCOMES],
+    },
+  ];
+}
+
+export function buildConfigureFunnelSeries(
+  steps: readonly ConfigureFunnelStepFilter[],
+): Array<{
+  kind: "EventsNode";
+  event: "p_dev_configure_step_completed";
+  name: string;
+  properties: ReturnType<typeof configureStepEventProperties>;
+}> {
+  return steps.map((step) => ({
+    kind: "EventsNode" as const,
+    event: "p_dev_configure_step_completed" as const,
+    name: step.stepId,
+    properties: configureStepEventProperties(step.stepId),
+  }));
+}

package/workspace-snapshot/files/src/observability/privacy-schema.ts

@@ -0,0 +1,229 @@
+import type { AnalyticsEvent, DurationBucket } from "./types.js";
+
+export const ANALYTICS_EVENT_NAMES = [
+  "p_dev_session_started",
+  "p_dev_configure_step_viewed",
+  "p_dev_configure_step_completed",
+  "p_dev_workspace_provision_started",
+  "p_dev_workspace_provision_completed",
+  "p_dev_workspace_provision_failed",
+  "p_dev_setup_completed",
+] as const;
+
+export const COMMON_ANALYTICS_PROPERTY_KEYS = [
+  "observability_schema_version",
+  "package_version",
+  "release_sha",
+  "runtime_mode",
+  "os_family",
+  "cpu_arch_family",
+  "node_major_version",
+  "session_id",
+  "first_launch_for_p_dev_home",
+  "workspace_kind",
+  "distinct_id",
+  "$process_person_profile",
+] as const;
+
+export const FORBIDDEN_PROPERTY_KEY_PATTERNS = [
+  /token/i,
+  /secret/i,
+  /password/i,
+  /authorization/i,
+  /cookie/i,
+  /email/i,
+  /repo/i,
+  /repository/i,
+  /github/i,
+  /linear/i,
+  /cursor/i,
+  /vercel/i,
+  /hostname/i,
+  /user_agent/i,
+  /user-agent/i,
+  /locale/i,
+  /timezone/i,
+  /prompt/i,
+  /source_code/i,
+  /path/i,
+  /url/i,
+  /query/i,
+  /operation_id/i,
+  /snapshot_id/i,
+] as const;
+
+export const ALLOWED_SENTRY_TAG_KEYS = [
+  "observability_schema_version",
+  "package_version",
+  "release_sha",
+  "session_id",
+  "runtime_mode",
+  "os_family",
+  "cpu_arch_family",
+  "node_major_version",
+  "lifecycle_phase",
+  "configure_step_id",
+  "product_error_code",
+  "error_category",
+  "operation_resumed",
+  "remote_mutation_begun",
+  "durable_recovery_state_exists",
+  "duration_bucket",
+  "retry_count_bucket",
+  "rate_limit_pause_count_bucket",
+] as const;
+
+export function bucketDurationMs(durationMs: number): DurationBucket {
+  if (!Number.isFinite(durationMs) || durationMs < 0) {
+    return "unknown";
+  }
+  if (durationMs < 10_000) {
+    return "lt_10s";
+  }
+  if (durationMs < 30_000) {
+    return "10s_30s";
+  }
+  if (durationMs < 120_000) {
+    return "30s_2m";
+  }
+  if (durationMs < 300_000) {
+    return "2m_5m";
+  }
+  return "gt_5m";
+}
+
+export function bucketProvisioningDurationMs(
+  durationMs: number,
+): DurationBucket {
+  if (!Number.isFinite(durationMs) || durationMs < 0) {
+    return "unknown";
+  }
+  if (durationMs < 60_000) {
+    return "lt_1m";
+  }
+  if (durationMs < 180_000) {
+    return "1m_3m";
+  }
+  if (durationMs < 600_000) {
+    return "3m_10m";
+  }
+  return "gt_10m";
+}
+
+export function bucketCount(value: number): string {
+  if (!Number.isFinite(value) || value < 0) {
+    return "unknown";
+  }
+  if (value === 0) {
+    return "0";
+  }
+  if (value === 1) {
+    return "1";
+  }
+  if (value <= 3) {
+    return "2_3";
+  }
+  return "gt_3";
+}
+
+export function bucketSnapshotFileCount(count: number): string {
+  if (!Number.isFinite(count) || count < 0) {
+    return "unknown";
+  }
+  if (count < 100) {
+    return "lt_100";
+  }
+  if (count < 500) {
+    return "100_499";
+  }
+  if (count < 1000) {
+    return "500_999";
+  }
+  return "gte_1000";
+}
+
+export function assertAllowedPropertyKeys(
+  properties: Record<string, unknown>,
+  allowedKeys: readonly string[],
+): void {
+  const allowed = new Set<string>(allowedKeys);
+  for (const key of Object.keys(properties)) {
+    if (!allowed.has(key)) {
+      throw new Error(`Observability property "${key}" is not allowlisted.`);
+    }
+    if (FORBIDDEN_PROPERTY_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
+      throw new Error(`Observability property "${key}" is forbidden.`);
+    }
+  }
+}
+
+export function analyticsEventToProperties(
+  event: AnalyticsEvent,
+): Record<string, unknown> {
+  switch (event.type) {
+    case "p_dev_session_started":
+      return {};
+    case "p_dev_configure_step_viewed":
+      return {
+        step_id: event.stepId,
+        step_number: event.stepNumber,
+        resumed: event.resumed,
+        revisited: event.revisited,
+      };
+    case "p_dev_configure_step_completed":
+      return {
+        step_id: event.stepId,
+        step_number: event.stepNumber,
+        resumed: event.resumed,
+        revisited: event.revisited,
+        duration_bucket: event.durationBucket,
+        completion_outcome: event.completionOutcome,
+      };
+    case "p_dev_workspace_provision_started":
+      return {
+        snapshot_file_count_bucket: event.snapshotFileCountBucket,
+        resumed_from_durable_pending_state:
+          event.resumedFromDurablePendingState,
+      };
+    case "p_dev_workspace_provision_completed":
+      return {
+        snapshot_file_count_bucket: event.snapshotFileCountBucket,
+        duration_bucket: event.durationBucket,
+        retry_count_bucket: event.retryCountBucket,
+        rate_limit_pause_count_bucket: event.rateLimitPauseCountBucket,
+        outcome: event.outcome,
+        resumed_from_durable_pending_state:
+          event.resumedFromDurablePendingState,
+        connected_to_existing_legacy_workspace:
+          event.connectedToExistingLegacyWorkspace,
+        created_snapshot_backed_workspace: event.createdSnapshotBackedWorkspace,
+      };
+    case "p_dev_workspace_provision_failed":
+      return {
+        snapshot_file_count_bucket: event.snapshotFileCountBucket,
+        duration_bucket: event.durationBucket,
+        retry_count_bucket: event.retryCountBucket,
+        rate_limit_pause_count_bucket: event.rateLimitPauseCountBucket,
+        failure_category: event.failureCategory,
+        resumed_from_durable_pending_state:
+          event.resumedFromDurablePendingState,
+        recovery_state_remained_after_failure:
+          event.recoveryStateRemainedAfterFailure,
+      };
+    case "p_dev_setup_completed":
+      return {};
+    default: {
+      const exhaustive: never = event;
+      throw new Error(`Unsupported analytics event: ${String(exhaustive)}`);
+    }
+  }
+}
+
+export function allowedAnalyticsPropertyKeysForEvent(
+  event: AnalyticsEvent,
+): string[] {
+  return [
+    ...COMMON_ANALYTICS_PROPERTY_KEYS,
+    ...Object.keys(analyticsEventToProperties(event)),
+  ];
+}

package/workspace-snapshot/files/src/observability/product-error-messages.ts

@@ -0,0 +1,19 @@
+const APPROVED_PRODUCT_ERROR_MESSAGES: Record<string, string> = {
+  uncaught_exception: "An unexpected uncaught exception occurred.",
+  unhandled_rejection: "An unexpected unhandled rejection occurred.",
+  configure_gui_spawn_error: "The Configure GUI process failed to start.",
+  configure_gui_health_check_failed:
+    "The Configure GUI failed its startup health check.",
+  configure_request_error: "A Configure request failed unexpectedly.",
+  provision_failed: "Harness workspace provisioning failed.",
+  p_dev_launch_failed: "The p-dev launcher failed during startup.",
+  harness_repo_provisioning_route_failed:
+    "Harness workspace provisioning failed.",
+};
+
+export function approvedProductErrorMessage(productErrorCode: string): string {
+  return (
+    APPROVED_PRODUCT_ERROR_MESSAGES[productErrorCode] ??
+    `Product error: ${productErrorCode}`
+  );
+}

package/workspace-snapshot/files/src/observability/product-errors.ts

@@ -0,0 +1,139 @@
+import type { HarnessProvisioningState } from "../setup/harness-repo-provisioning.js";
+import type { SnapshotProvisioningErrorCode } from "../setup/harness-snapshot-provisioning-helpers.js";
+import type { ErrorCategory, ProvisioningFailureCategory } from "./types.js";
+
+const AUTH_STATES = new Set<HarnessProvisioningState>([
+  "token-unavailable",
+  "token-invalid",
+  "token-unsupported",
+  "token-scope-ambiguous",
+  "token-insufficient",
+]);
+
+const PERMISSION_STATES = new Set<HarnessProvisioningState>([
+  "same-name-public-collision",
+  "same-name-unmanaged-collision",
+  "same-name-malformed-marker",
+]);
+
+const SNAPSHOT_STATES = new Set<HarnessProvisioningState>([
+  "snapshot-unavailable",
+  "snapshot-manifest-missing",
+  "snapshot-manifest-invalid",
+  "snapshot-incompatible",
+  "snapshot-tampered",
+  "snapshot-preview-stale",
+]);
+
+const LOCAL_PERSISTENCE_STATES = new Set<HarnessProvisioningState>([
+  "created-but-persistence-failed",
+]);
+
+export function mapHarnessProvisioningStateToErrorCategory(
+  state: HarnessProvisioningState,
+): ErrorCategory {
+  if (AUTH_STATES.has(state)) {
+    return "auth";
+  }
+  if (PERMISSION_STATES.has(state)) {
+    return "permission";
+  }
+  if (SNAPSHOT_STATES.has(state)) {
+    return "snapshot_validation";
+  }
+  if (LOCAL_PERSISTENCE_STATES.has(state)) {
+    return "local_persistence";
+  }
+  if (state.includes("rate-limit")) {
+    return "rate_limit";
+  }
+  if (state.includes("network")) {
+    return "network";
+  }
+  if (state.includes("conflict")) {
+    return "conflict";
+  }
+  return "unknown";
+}
+
+export function mapHarnessProvisioningStateToFailureCategory(
+  state: HarnessProvisioningState,
+): ProvisioningFailureCategory {
+  const category = mapHarnessProvisioningStateToErrorCategory(state);
+  if (category === "validation" || category === "unexpected") {
+    return "unknown";
+  }
+  return category;
+}
+
+export function mapSnapshotProvisioningErrorCode(
+  code: SnapshotProvisioningErrorCode,
+): ErrorCategory {
+  switch (code) {
+    case "repository-create-ambiguous":
+    case "commit-create-ambiguous":
+      return "conflict";
+    case "repository-create-reconciliation-failed":
+    case "repository-identity-mismatch":
+    case "ref-update-unexpected-head":
+    case "marker-commit-failed":
+    case "description-finalization-failed":
+      return "server";
+    case "snapshot-tree-mismatch":
+      return "snapshot_validation";
+    default:
+      return "unknown";
+  }
+}
+
+export function isUserCorrectableProvisioningState(
+  state: HarnessProvisioningState,
+): boolean {
+  return (
+    state === "explicit-repo-present" ||
+    state === "explicit-packaged-repo-invalid" ||
+    state === "explicit-packaged-repo-legacy-source" ||
+    state === "snapshot-preview-ready" ||
+    state === "repo-absent" ||
+    state === "valid-existing-managed-repo" ||
+    state === "skipped-not-packaged" ||
+    state === "skipped-source-mode"
+  );
+}
+
+export function shouldCaptureProvisioningStateAsProductError(
+  state: HarnessProvisioningState,
+): boolean {
+  return !isUserCorrectableProvisioningState(state);
+}
+
+export function productErrorCodeFromProvisioningState(
+  state: HarnessProvisioningState,
+): string {
+  return `harness_provisioning_${state.replace(/-/g, "_")}`;
+}
+
+export function productErrorCodeFromSnapshotCode(
+  code: SnapshotProvisioningErrorCode,
+): string {
+  return `snapshot_provisioning_${code.replace(/-/g, "_")}`;
+}
+
+export function mapHttpStatusToErrorCategory(status: number): ErrorCategory {
+  if (status === 401 || status === 403) {
+    return status === 401 ? "auth" : "permission";
+  }
+  if (status === 409) {
+    return "conflict";
+  }
+  if (status === 429) {
+    return "rate_limit";
+  }
+  if (status >= 500) {
+    return "server";
+  }
+  if (status >= 400) {
+    return "validation";
+  }
+  return "unknown";
+}

package/workspace-snapshot/files/src/observability/provisioning-analytics.ts

@@ -0,0 +1,28 @@
+import type { HarnessRepoProvisioningPreview } from "../setup/harness-repo-provisioning.js";
+import type { HarnessRepoProvisioningApplyResult } from "../setup/harness-repo-provisioning.js";
+
+export function deriveConnectedToExistingManagedWorkspace(input: {
+  persisted: boolean;
+  previewState: HarnessRepoProvisioningPreview["state"];
+}): boolean {
+  return (
+    input.persisted && input.previewState === "valid-existing-managed-repo"
+  );
+}
+
+export function deriveCreatedSnapshotBackedWorkspace(input: {
+  persisted: boolean;
+  applyState: HarnessRepoProvisioningApplyResult["state"];
+  preview: Pick<
+    HarnessRepoProvisioningPreview,
+    "willCreateRepository" | "state"
+  >;
+}): boolean {
+  if (!input.persisted || input.applyState !== "verified-and-persisted") {
+    return false;
+  }
+  return (
+    input.preview.willCreateRepository ||
+    input.preview.state === "same-name-snapshot-only-with-pending"
+  );
+}