@revos/cli 0.1.2 → 0.1.4

package/dist/templates/skills/create-semantic-model/references/cube-examples.md

@@ -0,0 +1,267 @@
+# Cube Overlay Examples
+
+## Table of Contents
+
+- [Standard Cube](#standard-cube)
+- [Bridge / Junction Cube](#bridge--junction-cube)
+- [Composite Primary Key](#composite-primary-key)
+- [Join Direction Examples](#join-direction-examples)
+- [Refresh Key Variants](#refresh-key-variants)
+- [Type Mapping](#type-mapping)
+- [Measure Suggestions](#measure-suggestions)
+
+---
+
+## Standard Cube
+
+```yaml
+name: hubspot_companies
+sql_table: "`revos_1737556292084.gold_hubspot_companies`"
+
+joins:
+  companies_deals:
+    sql: "${CUBE}.id = ${companies_deals}.company_id"
+    relationship: one_to_many
+
+measures:
+  count:
+    type: count
+
+  total_deal_value:
+    sql: "${CUBE}.properties_hs_total_deal_value"
+    type: sum
+
+dimensions:
+  id:
+    sql: "${CUBE}.id"
+    type: string
+    primary_key: true
+
+  airbyte_extracted_at:
+    sql: "${CUBE}._airbyte_extracted_at"
+    type: time
+
+refresh_key:
+  sql: "SELECT MAX(_airbyte_extracted_at) FROM `revos_1737556292084.gold_hubspot_companies`"
+```
+
+Notes:
+
+1. Cube `name` is `hubspot_companies` (no `gold_` prefix).
+2. `sql_table` references `gold_hubspot_companies` (with `gold_` prefix), in backticks.
+3. The join references `${companies_deals}` — the cube name of a bridge cube defined in `semantic/companies_deals.yml`.
+4. Only `_airbyte_extracted_at` is exposed from Airbyte metadata, as `airbyte_extracted_at`.
+5. `refresh_key.sql` uses the same fully qualified table name as `sql_table`.
+
+---
+
+## Bridge / Junction Cube
+
+```yaml
+name: companies_deals
+sql_table: "`revos_1737556292084.gold_companies_deals`"
+public: false
+
+joins:
+  hubspot_companies:
+    relationship: many_to_one
+    sql: "${CUBE}.company_id = ${hubspot_companies}.id"
+
+  hubspot_deals:
+    relationship: many_to_one
+    sql: "${CUBE}.deal_id = ${hubspot_deals}.id"
+
+measures:
+  count:
+    type: count
+
+dimensions:
+  id:
+    sql: "CONCAT(${CUBE}.deal_id, '-', ${CUBE}.company_id)"
+    type: string
+    primary_key: true
+
+  deal_id:
+    sql: "${CUBE}.deal_id"
+    type: string
+
+  company_id:
+    sql: "${CUBE}.company_id"
+    type: string
+
+  airbyte_extracted_at:
+    sql: "${CUBE}._airbyte_extracted_at"
+    type: time
+
+refresh_key:
+  sql: "SELECT MAX(_airbyte_extracted_at) FROM `revos_1737556292084.gold_companies_deals`"
+```
+
+If the bridge model lacks `_airbyte_extracted_at`, omit that dimension and use:
+
+```yaml
+refresh_key:
+  every: 1 hour
+```
+
+---
+
+## Composite Primary Key
+
+Cube allows exactly one `primary_key: true` per cube. For composite keys, create a synthetic dimension:
+
+```yaml
+dimensions:
+  id:
+    sql: "CONCAT(${CUBE}.office_unique_id, '-', ${CUBE}.month)"
+    type: string
+    primary_key: true
+
+  office_unique_id:
+    sql: "${CUBE}.office_unique_id"
+    type: string
+
+  month:
+    sql: "${CUBE}.month"
+    type: time
+```
+
+Choose a separator that does not appear in component values. `-` is usually safe; use `||` if components may contain `-`.
+
+Joins to this cube must reference the synthetic `id`, not individual components.
+
+---
+
+## Join Direction Examples
+
+Direction is always from the perspective of the current cube.
+
+### Direct many-to-one / one-to-many
+
+```yaml
+# In hubspot_deals.yml
+joins:
+  hubspot_companies:
+    sql: "${CUBE}.company_id = ${hubspot_companies}.id"
+    relationship: many_to_one
+```
+
+```yaml
+# In hubspot_companies.yml
+joins:
+  hubspot_deals:
+    sql: "${CUBE}.id = ${hubspot_deals}.company_id"
+    relationship: one_to_many
+```
+
+### Connector path (products -> clients -> addresses)
+
+```yaml
+# In products.yml
+joins:
+  clients:
+    sql: "${CUBE}.client_id = ${clients}.id"
+    relationship: many_to_one
+```
+
+```yaml
+# In clients.yml
+joins:
+  products:
+    sql: "${CUBE}.id = ${products}.client_id"
+    relationship: one_to_many
+  addresses:
+    sql: "${CUBE}.id = ${addresses}.client_id"
+    relationship: one_to_many
+```
+
+```yaml
+# In addresses.yml
+joins:
+  clients:
+    sql: "${CUBE}.client_id = ${clients}.id"
+    relationship: many_to_one
+```
+
+### Bridge joins (both parents reference bridge)
+
+```yaml
+# In hubspot_companies.yml
+joins:
+  companies_deals:
+    sql: "${CUBE}.id = ${companies_deals}.company_id"
+    relationship: one_to_many
+```
+
+```yaml
+# In hubspot_deals.yml
+joins:
+  companies_deals:
+    sql: "${CUBE}.id = ${companies_deals}.deal_id"
+    relationship: one_to_many
+```
+
+### Unvalidated join
+
+```yaml
+joins:
+  hubspot_companies:
+    # UNVALIDATED: match rate could not be measured because gold_hubspot_companies was not yet materialized
+    sql: "${CUBE}.company_id = ${hubspot_companies}.id"
+    relationship: many_to_one
+```
+
+---
+
+## Refresh Key Variants
+
+Priority order:
+
+```yaml
+# 1. Airbyte timestamp (preferred)
+refresh_key:
+  sql: "SELECT MAX(_airbyte_extracted_at) FROM `<dataset>.<gold_model>`"
+
+# 2. Other reliable timestamp
+refresh_key:
+  sql: "SELECT MAX(updated_at) FROM `<dataset>.<gold_model>`"
+
+# 3. Default fallback
+refresh_key:
+  every: 1 hour
+```
+
+`refresh_key.sql` must reference the same fully qualified table as the cube's `sql_table`.
+
+---
+
+## Type Mapping
+
+```text
+STRING / VARCHAR / TEXT        -> string
+INTEGER / FLOAT / NUMERIC      -> number
+BOOLEAN / BOOL                 -> boolean
+DATE / DATETIME / TIMESTAMP    -> time
+JSON / ARRAY / STRUCT          -> string (or skip if not queryable)
+```
+
+---
+
+## Measure Suggestions
+
+Common measure patterns by column name:
+
+```text
+amount   -> total_amount (sum), average_amount (avg)
+revenue  -> total_revenue (sum)
+price    -> total_price or average_price
+cost     -> total_cost (sum)
+quantity -> total_quantity (sum)
+duration -> average_duration (avg)
+*_id     -> count_distinct (only in the cube that owns the FK)
+created_at  -> first_created_at (min), last_created_at (max)
+closed_at   -> first_closed_at (min), last_closed_at (max)
+updated_at  -> last_updated_at (max)
+```
+
+`count_distinct` on FK columns: define inside the cube that owns the FK, not the parent cube. Joins can produce row fan-out that distorts distinct counts.

package/dist/templates/skills/create-semantic-model/references/key-patterns.md

@@ -0,0 +1,150 @@
+# Key Detection Patterns
+
+## Table of Contents
+
+- [Primary Key Patterns](#primary-key-patterns)
+- [Secondary Key Patterns](#secondary-key-patterns)
+- [Foreign Key Patterns](#foreign-key-patterns)
+- [JSON / Array Key Patterns](#json--array-key-patterns)
+- [Schema Summary Output](#schema-summary-output)
+
+---
+
+## Primary Key Patterns
+
+Common primary key column names:
+
+```text
+id
+<entity>_id
+<model_name>_id
+uuid
+unique_id
+external_id
+source_id
+```
+
+Examples:
+
+```text
+companies.id
+companies.company_id
+companies.office_unique_id
+hubspot_companies.properties_company_unique_id
+```
+
+Rules:
+
+1. Prefer a known business or platform identifier over a generated row number.
+2. Prefer stable IDs over names or labels.
+3. Do not mark a column as primary key only because it looks unique by name.
+4. Validate uniqueness with SQL (see references/validation-queries.md, section 1).
+
+---
+
+## Secondary Key Patterns
+
+Secondary keys are identifiers that are not the table primary key but can be used for joins, grouping, lookup, or `count_distinct` measures.
+
+Common patterns:
+
+```text
+office_unique_id
+company_id
+customer_id
+client_id
+deal_id
+contact_id
+owner_id
+user_id
+account_id
+product_id
+address_id
+external_id
+source_id
+```
+
+Rules:
+
+1. Track secondary keys explicitly.
+2. Secondary keys may be foreign keys to another entity.
+3. Secondary keys should usually become Cube dimensions.
+4. Secondary keys may support `count_distinct` measures if analytically useful, but only inside the cube that owns the FK (see Phase 7 caution about fan-out).
+
+---
+
+## Foreign Key Patterns
+
+Common patterns:
+
+```text
+<entity>_id
+<entity>Id
+fk_<entity>
+associated_<entity>_id
+parent_<entity>_id
+owner_id
+created_by_user_id
+updated_by_user_id
+```
+
+Also check JSON and array-based foreign keys:
+
+```text
+deals.companies -> companies.id
+companies.deals -> deals.id
+contacts.associated_company_ids -> companies.id
+```
+
+---
+
+## JSON / Array Key Patterns
+
+Keys may be hidden inside JSON strings, JSON arrays, repeated fields, or nested structures. This is especially common for one-to-many and many-to-many relationships.
+
+Common column names that may contain relationship keys:
+
+```text
+companies, deals, contacts, users, owners, clients, products, addresses
+associations, associated_companies, associated_deals, associated_contacts
+associated_clients, associated_products
+company_ids, deal_ids, contact_ids, client_ids, product_ids, address_ids
+```
+
+Example: `gold_hubspot_deals.companies` may contain an array of company IDs.
+
+For JSON arrays, use `UNNEST(JSON_VALUE_ARRAY(...))`.
+
+Rules:
+
+1. Always inspect JSON, array, repeated, and nested fields for hidden relationship keys.
+2. Do not assume relationship keys only exist as flat columns.
+3. If JSON structure is unknown, inspect sample values first:
+
+```sql
+SELECT <json_or_array_column>
+FROM `<dataset>.<gold_model>`
+WHERE <json_or_array_column> IS NOT NULL
+LIMIT 20;
+```
+
+4. If a relationship is stored as an array of IDs, use or create an approved bridge/support model. Bridge model creation is delegated to `create-dbt-transformations`.
+5. Bridge models should preserve both sides of the relationship as keys.
+6. Bridge and junction cubes should use `public: false` where the project convention supports it.
+
+---
+
+## Schema Summary Output
+
+After analysis, summarize each selected model in this format:
+
+```text
+Model: gold_hubspot_deals
+Columns: 18
+Candidate primary key: deal_id
+Secondary keys: company_id, owner_id
+JSON / array relationship columns: companies, contacts
+Time columns: created_at, updated_at, closed_at
+Numeric metric-like columns: amount
+Airbyte columns present: _airbyte_extracted_at (will be exposed), _airbyte_raw_id, _airbyte_meta, _airbyte_generation_id (will be excluded by default)
+```

package/dist/templates/skills/create-semantic-model/references/validation-queries.md

@@ -0,0 +1,209 @@
+# Join Validation SQL Templates
+
+All queries use the literal `<dataset>` value resolved at the start of Phase 2. Substitute the placeholder before executing.
+
+## Table of Contents
+
+- [1. Key Uniqueness](#1-key-uniqueness)
+- [2. Many-to-One Direction](#2-many-to-one-direction)
+- [3. Reverse One-to-Many Direction](#3-reverse-one-to-many-direction)
+- [4. One-to-One Relationships](#4-one-to-one-relationships)
+- [5. Many-to-Many Through Bridge](#5-many-to-many-through-bridge)
+- [6. JSON / Array Relationships](#6-json--array-relationships)
+- [7. Type Compatibility](#7-type-compatibility)
+
+---
+
+## 1. Key Uniqueness
+
+```sql
+SELECT
+  COUNT(*) AS total_rows,
+  COUNT(DISTINCT <candidate_pk>) AS distinct_keys,
+  COUNT(*) - COUNT(DISTINCT <candidate_pk>) AS duplicate_count
+FROM `<dataset>.<gold_model>`;
+```
+
+A primary key should have `duplicate_count = 0`. If duplicates exist, do not mark the column as `primary_key: true` unless clearly documented.
+
+---
+
+## 2. Many-to-One Direction
+
+Example: `deals.company_id -> companies.id` (many_to_one).
+
+Validate FK match rate:
+
+```sql
+SELECT
+  COUNT(*) AS total_rows_with_fk,
+  COUNT(c.id) AS matched_rows,
+  COUNT(*) - COUNT(c.id) AS unmatched_rows,
+  ROUND(100.0 * COUNT(c.id) / COUNT(*), 2) AS match_percentage
+FROM `<dataset>.gold_hubspot_deals` d
+LEFT JOIN `<dataset>.gold_hubspot_companies` c
+  ON d.company_id = c.id
+WHERE d.company_id IS NOT NULL;
+```
+
+Check for fan-out (should be empty for valid many-to-one):
+
+```sql
+SELECT
+  d.deal_id,
+  COUNT(c.id) AS matched_companies
+FROM `<dataset>.gold_hubspot_deals` d
+LEFT JOIN `<dataset>.gold_hubspot_companies` c
+  ON d.company_id = c.id
+WHERE d.company_id IS NOT NULL
+GROUP BY d.deal_id
+HAVING COUNT(c.id) > 1
+LIMIT 20;
+```
+
+---
+
+## 3. Reverse One-to-Many Direction
+
+Validate reverse aggregation:
+
+```sql
+SELECT
+  c.id AS company_id,
+  COUNT(d.deal_id) AS deal_count
+FROM `<dataset>.gold_hubspot_companies` c
+LEFT JOIN `<dataset>.gold_hubspot_deals` d
+  ON c.id = d.company_id
+GROUP BY c.id
+ORDER BY deal_count DESC
+LIMIT 20;
+```
+
+Cross-check sampled counts:
+
+```sql
+SELECT
+  company_id,
+  COUNT(*) AS expected_deal_count
+FROM `<dataset>.gold_hubspot_deals`
+WHERE company_id IN (<sample_company_ids>)
+GROUP BY company_id;
+```
+
+Counts must match.
+
+---
+
+## 4. One-to-One Relationships
+
+Validate uniqueness on both sides, then validate the join:
+
+```sql
+SELECT
+  COUNT(*) AS total_rows,
+  COUNT(r.<right_key>) AS matched_rows,
+  COUNT(*) - COUNT(r.<right_key>) AS unmatched_rows,
+  ROUND(100.0 * COUNT(r.<right_key>) / COUNT(*), 2) AS match_percentage
+FROM `<dataset>.<left_model>` l
+LEFT JOIN `<dataset>.<right_model>` r
+  ON l.<left_key> = r.<right_key>
+WHERE l.<left_key> IS NOT NULL;
+```
+
+Also validate reverse. If either side has duplicate keys, the relationship is not one-to-one.
+
+---
+
+## 5. Many-to-Many Through Bridge
+
+Validate both bridge edges. Example: `companies <-> deals through gold_companies_deals`.
+
+Bridge to one parent:
+
+```sql
+SELECT
+  COUNT(*) AS total_bridge_rows,
+  COUNT(c.id) AS matched_companies,
+  COUNT(*) - COUNT(c.id) AS unmatched_companies,
+  ROUND(100.0 * COUNT(c.id) / COUNT(*), 2) AS match_percentage
+FROM `<dataset>.gold_companies_deals` b
+LEFT JOIN `<dataset>.gold_hubspot_companies` c
+  ON b.company_id = c.id
+WHERE b.company_id IS NOT NULL;
+```
+
+Run analogous query for the other parent. Then validate reverse aggregations:
+
+```sql
+SELECT
+  c.id AS company_id,
+  COUNT(b.deal_id) AS related_deals
+FROM `<dataset>.gold_hubspot_companies` c
+LEFT JOIN `<dataset>.gold_companies_deals` b
+  ON c.id = b.company_id
+GROUP BY c.id
+ORDER BY related_deals DESC
+LIMIT 20;
+```
+
+Same query swapped for deals -> bridge -> companies. Report sampled counts.
+
+---
+
+## 6. JSON / Array Relationships
+
+Validate extracted keys:
+
+```sql
+WITH extracted AS (
+  SELECT DISTINCT
+    src.<source_pk> AS source_id,
+    extracted_id
+  FROM `<dataset>.<source_model>` src,
+  UNNEST(JSON_VALUE_ARRAY(src.<json_array_column>)) AS extracted_id
+)
+SELECT
+  COUNT(*) AS total_relationships,
+  COUNT(tgt.<target_pk>) AS matched_relationships,
+  COUNT(*) - COUNT(tgt.<target_pk>) AS unmatched_relationships,
+  ROUND(100.0 * COUNT(tgt.<target_pk>) / COUNT(*), 2) AS match_percentage
+FROM extracted e
+LEFT JOIN `<dataset>.<target_model>` tgt
+  ON e.extracted_id = tgt.<target_pk>;
+```
+
+Sample matched values:
+
+```sql
+WITH extracted AS (
+  SELECT DISTINCT
+    src.<source_pk> AS source_id,
+    extracted_id
+  FROM `<dataset>.<source_model>` src,
+  UNNEST(JSON_VALUE_ARRAY(src.<json_array_column>)) AS extracted_id
+)
+SELECT
+  e.source_id, e.extracted_id, tgt.<target_pk>, tgt.<display_column>
+FROM extracted e
+LEFT JOIN `<dataset>.<target_model>` tgt
+  ON e.extracted_id = tgt.<target_pk>
+LIMIT 10;
+```
+
+---
+
+## 7. Type Compatibility
+
+```sql
+SELECT column_name, data_type
+FROM `<dataset>.INFORMATION_SCHEMA.COLUMNS`
+WHERE table_name IN ('<source_model>', '<target_model>')
+  AND column_name IN ('<foreign_key>', '<target_pk>');
+```
+
+If types differ:
+
+1. Report the mismatch.
+2. Prefer fixing type alignment in the dbt model or approved support model.
+3. Only cast in Cube join SQL when necessary.
+4. Prefer casting the foreign-key side to match the primary-key side.

package/dist/templates/skills/explore-lakehouse/SKILL.md

@@ -1,6 +1,13 @@
 ---
 name: explore-lakehouse
-description: Explore the RevOS BigQuery lakehouse — list datasets, list tables, inspect table schemas, preview sample rows, assess data layer (bronze/silver/gold), and check data completeness/null rates. Use when asked to explore the lakehouse, understand available raw data, inspect table structure, assess data quality, or check what's been built in dbt.
+description: >
+  Inspect the RevOS BigQuery lakehouse: list datasets and tables, introspect table schemas
+  and column types, preview sample rows, assess data layers (bronze/silver/gold), and check
+  data completeness and null rates. Required companion skill for create-dbt-transformations
+  and create-semantic-model — load before generating dbt models or semantic overlays to
+  introspect warehouse columns and types. Use when asked to: explore the lakehouse, list
+  BigQuery tables, inspect a table schema, preview data, check raw source tables, assess data
+  quality, check null rates, understand available data, or perform BigQuery schema introspection.
 ---
 
 # Explore Lakehouse

package/dist/{types-DZssnweO.d.mts → types-DmuJzN0Z.d.mts}

@@ -3,6 +3,10 @@ import { CreateOverlayDto, OverlaysResponseDto } from "@revos/api-client";
 //#region src/core/types.d.ts
 type CubeOverlay = OverlaysResponseDto["data"][number];
 type CubeDefinition = CubeOverlay["data"];
+type OverlayFileData = CubeDefinition & {
+  name?: string;
+  description?: string;
+};
 type OverlayFile = CreateOverlayDto;
 interface Config {
   apiUrl: string;
@@ -62,4 +66,4 @@ interface DiffResult {
   entries: DiffEntry[];
 }
 //#endregion
-export { DiffEntry as a, OverlayStatusInfo as c, StatusResult as d, SyncStatus as f, DiffChange as i, PullResult as l, CubeDefinition as n, DiffResult as o, CubeOverlay as r, OverlayFile as s, Config as t, PushResult as u };
+export { DiffEntry as a, OverlayFileData as c, PushResult as d, StatusResult as f, DiffChange as i, OverlayStatusInfo as l, CubeDefinition as n, DiffResult as o, SyncStatus as p, CubeOverlay as r, OverlayFile as s, Config as t, PullResult as u };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@revos/cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "RevOS CLI for managing RevOS platform resources",
   "type": "module",
   "main": "dist/index.js",
@@ -46,6 +46,7 @@
     "@oclif/table": "^0.5.4",
     "chalk": "^4.1.2",
     "open": "^10.1.0",
+    "yaml": "^2.8.3",
     "@revos/api-client": "0.1.0"
   },
   "devDependencies": {