kpi_assembler 0.5.0

data/lib/kpi_assembler/schema_kpi_proposer.rb

@@ -0,0 +1,323 @@
+# frozen_string_literal: true
+
+module KPIAssembler
+  # Builds candidate KPI recipes from whatever tables were introspected.
+  # Recognized schema patterns are overlays on top of generic volume, rate,
+  # and revenue guesses.
+  class SchemaKpiProposer
+    SUCCESS_STATUSES = /converted|won|sold|closed|paid|complete|active|member/i.freeze
+    MONEY = /amount|price|total|revenue|fee|mrr|sales_amount/i.freeze
+    LOCATION = %w[location_id company_id site_id club_id].freeze
+
+    def initialize(schema, options = {})
+      @schema = schema
+      @options = options
+    end
+
+    def propose
+      candidates = []
+      candidates.concat(sample_funnel_candidates) if sample_funnel_schema?
+      candidates.concat(people_activity_candidates) if people_activity_schema?
+      candidates.concat(generic_candidates)
+      candidates.uniq { |kpi| kpi[:id] }.first(14)
+    end
+
+    private
+
+    def sample_funnel_schema?
+      %w[leads appointments memberships invoices locations].all? { |table| @schema.key?(table) }
+    end
+
+    def people_activity_schema?
+      table_has?("people", %w[created_at]) && table_has?("activities", %w[start])
+    end
+
+    def sample_funnel_candidates
+      [
+        kpi(
+          id: "lead_conversion_rate",
+          name: "Lead to Member Conversion Rate",
+          category: "Sales Funnel",
+          owner: "sales",
+          english: "Share of leads created in the period that converted to a membership.",
+          sql: period_sql("leads", "created_at", "(CAST(COUNT(CASE WHEN status = 'converted' THEN 1 END) AS FLOAT) / NULLIF(COUNT(*), 0)) * 100"),
+          grain: "daily",
+          location_column: "location_id",
+          fact_table: "leads"
+        ),
+        kpi(
+          id: "tour_show_rate",
+          name: "Tour Show Rate",
+          category: "Sales Funnel",
+          owner: "sales",
+          english: "Share of scheduled tours that were completed (shown), excluding cancellations from the denominator.",
+          sql: period_sql("appointments", "scheduled_at", "(CAST(COUNT(CASE WHEN status = 'completed' THEN 1 END) AS FLOAT) / NULLIF(COUNT(CASE WHEN status IN ('completed', 'no_show') THEN 1 END), 0)) * 100"),
+          grain: "daily",
+          location_column: "location_id",
+          fact_table: "appointments"
+        ),
+        kpi(
+          id: "no_show_rate",
+          name: "Tour No-Show Rate",
+          category: "Sales Funnel",
+          owner: "site_manager",
+          english: "Share of scheduled tours marked no_show among completed + no_show visits.",
+          sql: period_sql("appointments", "scheduled_at", "(CAST(COUNT(CASE WHEN status = 'no_show' THEN 1 END) AS FLOAT) / NULLIF(COUNT(CASE WHEN status IN ('completed', 'no_show') THEN 1 END), 0)) * 100"),
+          grain: "daily",
+          location_column: "location_id",
+          fact_table: "appointments"
+        ),
+        kpi(
+          id: "paid_mrr",
+          name: "Paid Monthly Recurring Revenue",
+          category: "Revenue",
+          owner: "exec",
+          english: "Sum of paid invoice amounts billed in the period.",
+          sql: period_sql("invoices", "billed_at", "COALESCE(SUM(amount), 0)", extra: "status = 'paid'"),
+          grain: "monthly",
+          location_column: "location_id",
+          fact_table: "invoices"
+        ),
+        kpi(
+          id: "avg_revenue_per_member",
+          name: "Average Revenue Per Member",
+          category: "Revenue",
+          owner: "exec",
+          english: "Paid invoice amount divided by distinct memberships billed in the period.",
+          sql: period_sql("invoices", "billed_at", "SUM(amount) / NULLIF(COUNT(DISTINCT membership_id), 0)", extra: "status = 'paid'"),
+          grain: "monthly",
+          location_column: "location_id",
+          fact_table: "invoices"
+        ),
+        kpi(
+          id: "active_memberships",
+          name: "Active Memberships",
+          category: "Retention",
+          owner: "exec",
+          english: "Memberships that started before period end and were not cancelled before period start.",
+          sql: "SELECT COUNT(*) AS metric_value FROM memberships WHERE started_at < ? AND (cancelled_at IS NULL OR cancelled_at >= ?)#{tenant_and("memberships")}",
+          grain: "daily",
+          location_column: location_column_for("memberships"),
+          fact_table: "memberships",
+          bind_style: "end_then_start"
+        ),
+        kpi(
+          id: "revenue_per_lead_unsafe",
+          name: "Revenue per Lead (unconstrained join)",
+          category: "Revenue",
+          owner: "finance",
+          english: "Invoice revenue divided by leads using an unconstrained join — expected to fail certification.",
+          sql: "SELECT SUM(invoices.amount) / COUNT(leads.id) AS metric_value FROM leads JOIN invoices WHERE leads.created_at >= ? AND leads.created_at < ?",
+          grain: "monthly",
+          location_column: "leads.location_id",
+          fact_table: "leads"
+        )
+      ]
+    end
+
+    def people_activity_candidates
+      out = []
+      people_loc = location_column_for("people")
+      activity_loc = location_column_for("activities")
+
+      out << kpi(
+        id: "new_leads",
+        name: "New leads",
+        category: "Sales Funnel",
+        owner: "sales",
+        english: "People records created in the period.",
+        sql: period_sql("people", "created_at", "COUNT(*)", extra: soft_delete("people")),
+        grain: "daily",
+        location_column: people_loc,
+        fact_table: "people"
+      )
+
+      if column?("people", "sale_at")
+        out << kpi(
+          id: "lead_close_rate",
+          name: "Lead close rate",
+          category: "Sales Funnel",
+          owner: "sales",
+          english: "Share of people created in the period that have a sale_at timestamp.",
+          sql: period_sql(
+            "people",
+            "created_at",
+            "(CAST(COUNT(CASE WHEN sale_at IS NOT NULL THEN 1 END) AS FLOAT) / NULLIF(COUNT(*), 0)) * 100",
+            extra: soft_delete("people")
+          ),
+          grain: "daily",
+          location_column: people_loc,
+          fact_table: "people"
+        )
+      end
+
+      extra = [soft_delete("activities"), "type = 'Event'"].compact.reject(&:empty?).join(" AND ")
+      extra = nil if extra.empty?
+      out << kpi(
+        id: "scheduled_appointments",
+        name: "Scheduled appointments",
+        category: "Sales Funnel",
+        owner: "sales",
+        english: "Sales appointments (activities.type = Event) starting in the period.",
+        sql: period_sql("activities", "start", "COUNT(*)", extra: extra),
+        grain: "daily",
+        location_column: activity_loc,
+        fact_table: "activities"
+      )
+
+      if column?("activities", "complete")
+        out << kpi(
+          id: "appointment_completion_rate",
+          name: "Appointment completion rate",
+          category: "Sales Funnel",
+          owner: "site_manager",
+          english: "Share of period appointments marked complete.",
+          sql: period_sql(
+            "activities",
+            "start",
+            "(CAST(COUNT(CASE WHEN complete THEN 1 END) AS FLOAT) / NULLIF(COUNT(*), 0)) * 100",
+            extra: extra
+          ),
+          grain: "daily",
+          location_column: activity_loc,
+          fact_table: "activities"
+        )
+      end
+
+      if column?("activities", "sales_amount")
+        out << kpi(
+          id: "appointment_sales_amount",
+          name: "Appointment sales amount",
+          category: "Revenue",
+          owner: "exec",
+          english: "Sum of sales_amount on appointments starting in the period.",
+          sql: period_sql("activities", "start", "COALESCE(SUM(sales_amount), 0)", extra: extra),
+          grain: "monthly",
+          location_column: activity_loc,
+          fact_table: "activities"
+        )
+      end
+
+      out
+    end
+
+    def generic_candidates
+      @schema.filter_map do |table, meta|
+        next if sample_funnel_schema? && %w[leads appointments memberships invoices locations].include?(table)
+        next if people_activity_schema? && %w[people activities companies].include?(table)
+
+        time = pick_time(meta)
+        next unless time
+
+        money = meta[:columns].map { |col| col[:name] }.find { |name| name.match?(MONEY) }
+        status_col = meta[:columns].map { |col| col[:name] }.find { |name| name.match?(/\A(status|state|outcome)\z/i) }
+        loc = location_column_for(table)
+        success = success_value(meta, status_col)
+
+        if money
+          extra = status_col && (meta.dig(:categorical_values, status_col) || []).any? { |value| value.to_s.match?(/paid/i) } ? "#{status_col} = 'paid'" : nil
+          kpi(
+            id: "#{table}_#{money}_sum",
+            name: "#{human(table)} #{human(money)}",
+            category: "Revenue",
+            owner: "exec",
+            english: "Sum of #{money} on #{table} in the period.",
+            sql: period_sql(table, time, "COALESCE(SUM(#{money}), 0)", extra: extra),
+            grain: "monthly",
+            location_column: loc,
+            fact_table: table
+          )
+        elsif success && status_col
+          kpi(
+            id: "#{table}_#{success}_rate",
+            name: "#{human(table)} #{human(success)} rate",
+            category: "Conversion",
+            owner: "ops",
+            english: "Share of #{table} rows whose #{status_col} is #{success}.",
+            sql: period_sql(
+              table,
+              time,
+              "(CAST(COUNT(CASE WHEN #{status_col} = '#{success}' THEN 1 END) AS FLOAT) / NULLIF(COUNT(*), 0)) * 100"
+            ),
+            grain: "daily",
+            location_column: loc,
+            fact_table: table
+          )
+        else
+          kpi(
+            id: "#{table}_volume",
+            name: "#{human(table)} volume",
+            category: "Operations",
+            owner: "ops",
+            english: "Row count of #{table} in the period.",
+            sql: period_sql(table, time, "COUNT(*)", extra: soft_delete(table)),
+            grain: "daily",
+            location_column: loc,
+            fact_table: table
+          )
+        end
+      end
+    end
+
+    def pick_time(meta)
+      preferred = %w[created_at billed_at scheduled_at started_at occurred_at start completed_at]
+      (preferred & meta[:time_columns]) .first || meta[:time_columns].first
+    end
+
+    def success_value(meta, status_col)
+      return unless status_col
+
+      Array(meta.dig(:categorical_values, status_col)).map(&:to_s).find { |value| value.match?(SUCCESS_STATUSES) }
+    end
+
+    def location_column_for(table)
+      names = Array(@schema.dig(table, :columns)).map { |col| col[:name] }
+      LOCATION.find { |column| names.include?(column) }
+    end
+
+    def table_has?(table, columns)
+      return false unless @schema.key?(table)
+
+      names = @schema[table][:columns].map { |col| col[:name] }
+      columns.all? { |column| names.include?(column) }
+    end
+
+    def column?(table, name)
+      Array(@schema.dig(table, :columns)).any? { |col| col[:name] == name }
+    end
+
+    def soft_delete(table)
+      if column?(table, "deleted_at")
+        "deleted_at IS NULL"
+      elsif column?(table, "deleted")
+        "deleted = false"
+      end
+    end
+
+    def tenant_clause(table)
+      column = @options[:tenant_column]
+      id = @options[:tenant_id]
+      return unless column && id && column?(table, column)
+
+      "#{column} = #{Integer(id)}"
+    end
+
+    def tenant_and(table)
+      clause = tenant_clause(table)
+      clause ? " AND #{clause}" : ""
+    end
+
+    def period_sql(table, time_column, expression, extra: nil)
+      clauses = ["#{time_column} >= ?", "#{time_column} < ?", extra, tenant_clause(table)].compact.reject { |part| part.to_s.empty? }
+      "SELECT #{expression} AS metric_value FROM #{table} WHERE #{clauses.join(" AND ")}"
+    end
+
+    def human(value)
+      value.to_s.tr("_", " ").gsub(/\b\w/, &:upcase)
+    end
+
+    def kpi(attrs)
+      attrs.merge(formula_description: attrs.delete(:english) || attrs[:formula_description])
+    end
+  end
+end

data/lib/kpi_assembler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module KPIAssembler
+  VERSION = "0.5.0"
+end

data/lib/kpi_assembler/web_app.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "sinatra/base"
+require_relative "../kpi_assembler"
+require_relative "connection"
+require_relative "web_service"
+
+module KPIAssembler
+  class WebApp < Sinatra::Base
+    set :root, File.expand_path("../..", __dir__)
+    set :public_folder, File.join(root, "public")
+    set :static, true
+    set :show_exceptions, false
+
+    configure do
+      connection = Connection.open
+      include_tables = ENV.fetch("KPI_INCLUDE_TABLES", "").split(",").map(&:strip).reject(&:empty?)
+      tenant_column = ENV["KPI_TENANT_COLUMN"]
+      tenant_id = ENV["KPI_TENANT_ID"]
+
+      set :web_service, WebService.new(
+        db: connection,
+        options: {
+          llm_provider: ENV.fetch("KPI_LLM_PROVIDER", ENV["GEMINI_API_KEY"] ? "gemini" : "ollama"),
+          use_llm: ENV["KPI_USE_LLM"] != "false",
+          use_ollama: ENV["KPI_USE_OLLAMA"] != "false",
+          ollama_model: ENV.fetch("KPI_OLLAMA_MODEL", "llama3"),
+          gemini_api_key: ENV["GEMINI_API_KEY"],
+          gemini_model: ENV.fetch("KPI_GEMINI_MODEL", "gemini-2.0-flash"),
+          source_name: connection.label,
+          include_tables: include_tables,
+          max_tables: ENV["KPI_MAX_TABLES"],
+          schema_name: ENV["KPI_DB_SCHEMA"],
+          tenant_column: tenant_column,
+          tenant_id: tenant_id && !tenant_id.empty? ? Integer(tenant_id) : nil
+        }
+      )
+    end
+
+    before "/api/*" do
+      content_type :json
+      headers(
+        "Access-Control-Allow-Origin" => ENV.fetch("KPI_ALLOWED_ORIGIN", "*"),
+        "Access-Control-Allow-Methods" => "GET, POST, OPTIONS",
+        "Access-Control-Allow-Headers" => "Content-Type, Authorization",
+        "Cache-Control" => "no-store"
+      )
+    end
+
+    options "/api/*" do
+      204
+    end
+
+    get "/" do
+      send_file File.join(settings.public_folder, "index.html")
+    end
+
+    get "/api/v1/health" do
+      json(
+        status: "ok",
+        service: "KPIAssembler",
+        version: KPIAssembler::VERSION,
+        source: settings.web_service.db.label,
+        dialect: settings.web_service.db.dialect,
+        principle: "The LLM proposes; deterministic code certifies."
+      )
+    end
+
+    post "/api/v1/discover" do
+      json(settings.web_service.discover)
+    end
+
+    get "/api/v1/discover" do
+      json(settings.web_service.discover)
+    end
+
+    post "/api/v1/certify" do
+      payload = parse_json_body
+      pack = settings.web_service.certify(payload["accepted_ids"])
+      json(pack.merge(locations: settings.web_service.locations))
+    end
+
+    get "/api/v1/pack" do
+      pack = settings.web_service.pack
+      halt 404, json(error: "No pack has been certified yet") unless pack
+
+      json(pack.merge(locations: settings.web_service.locations))
+    end
+
+    get "/api/v1/integration" do
+      json(
+        api_version: "v1",
+        discovery_endpoint: "/api/v1/discover",
+        certification_endpoint: "/api/v1/certify",
+        pack_endpoint: "/api/v1/pack",
+        embed_script: "/kpi-assembler.js",
+        database_configuration: {
+          url: "KPI_DATABASE_URL=postgres://user:pass@host:5432/application",
+          sqlite: "KPI_DATABASE_PATH=/path/to/application.sqlite3",
+          tables: "KPI_INCLUDE_TABLES=orders,customers,accounts",
+          tenant: "KPI_TENANT_COLUMN=account_id KPI_TENANT_ID=123"
+        }
+      )
+    end
+
+    not_found do
+      content_type :json if request.path.start_with?("/api/")
+      json(error: "Not found", path: request.path)
+    end
+
+    error KPIAssembler::Error do
+      status 422
+      json(error: env["sinatra.error"].message)
+    end
+
+    error JSON::ParserError do
+      status 400
+      json(error: "Request body must be valid JSON")
+    end
+
+    error do
+      error = env["sinatra.error"]
+      warn "[KPIAssembler] #{error.class}: #{error.message}"
+      status 500
+      json(error: "Unexpected server error")
+    end
+
+    private
+
+    def parse_json_body
+      raw = request.body.read
+      return {} if raw.empty?
+
+      JSON.parse(raw)
+    end
+
+    def json(payload)
+      JSON.generate(payload)
+    end
+  end
+end

data/lib/kpi_assembler/web_service.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "thread"
+require_relative "connection"
+require_relative "location_directory"
+
+module KPIAssembler
+  class WebService
+    attr_reader :db, :options
+
+    def initialize(db:, options: {})
+      @db = Connection.wrap(db)
+      @options = options
+      @mutex = Mutex.new
+      @schema = nil
+      @candidates = nil
+      @pack = nil
+    end
+
+    def discover
+      @mutex.synchronize { run_discovery }
+    end
+
+    def certify(accepted_ids)
+      @mutex.synchronize do
+        ids = Array(accepted_ids).map(&:to_s).uniq
+        raise Error, "Select at least one KPI to certify" if ids.empty?
+
+        # Discovery state lives in process memory, so a certify request can reach a
+        # worker that never served the matching discover request. Rebuild rather
+        # than rejecting a selection the user legitimately made.
+        run_discovery unless @schema && @candidates
+
+        known_ids = @candidates.map { |candidate| candidate[:id].to_s }
+        unknown_ids = ids - known_ids
+        raise Error, "Unknown KPI IDs: #{unknown_ids.join(', ')}" unless unknown_ids.empty?
+
+        selected = @candidates.select { |candidate| ids.include?(candidate[:id].to_s) }
+        now = options[:now] || Time.now.utc
+        certified, drafts = CertificationEngine.new(
+          db,
+          @schema,
+          now: now,
+          tenant_column: options[:tenant_column],
+          tenant_id: options[:tenant_id]
+        ).certify(selected)
+        evaluations = evaluate_views(certified, now)
+        briefing = BriefingGenerator.new(db, certified, now: now, tenant_id: options[:tenant_id]).generate
+
+        @pack = PackBuilder.new(
+          certified,
+          drafts: drafts,
+          briefing: briefing,
+          evaluations: evaluations,
+          schema: @schema
+        ).build
+      end
+    end
+
+    def pack
+      @mutex.synchronize { @pack }
+    end
+
+    def locations
+      LocationDirectory.new(db, tenant_id: options[:tenant_id]).all
+    end
+
+    private
+
+    def run_discovery
+      inspector = SchemaInspector.new(db, inspector_options)
+      @schema = inspector.introspect
+      generator = CandidateGenerator.new(@schema, generator_options)
+      @candidates = generator.generate
+      @pack = nil
+
+      {
+        source: options[:source_name] || db.label,
+        dialect: db.dialect,
+        schema_name: db.postgres? ? inspector_options[:schema_name] : nil,
+        tables: schema_summary,
+        missing_tables: inspector.missing_tables,
+        omitted_tables: inspector.omitted_tables,
+        max_tables: inspector_options[:max_tables],
+        proposer: generator.meta[:proposer],
+        proposer_model: generator.meta[:model],
+        proposer_fallback: generator.meta[:fallback_reason],
+        ranked_tables: generator.meta[:ranked_tables],
+        candidates: @candidates
+      }
+    end
+
+    def inspector_options
+      {
+        include_tables: Array(options[:include_tables]),
+        max_tables: options[:max_tables],
+        schema_name: options[:schema_name]
+      }
+    end
+
+    def generator_options
+      options.merge(dialect: db.dialect)
+    end
+
+    def schema_summary
+      @schema.map do |name, metadata|
+        {
+          name: name,
+          row_count: metadata[:row_count],
+          columns: metadata[:columns].map { |column| column.slice(:name, :type, :pk) },
+          foreign_keys: metadata[:foreign_keys],
+          time_columns: metadata[:time_columns]
+        }
+      end
+    end
+
+    def evaluate_views(certified, now)
+      evaluator = MetricEvaluator.new(db)
+      from = now - (30 * 86_400)
+      result = { "all" => values_for(evaluator, certified, from, now, nil) }
+      locations.each do |location|
+        result[location[:id].to_s] = values_for(evaluator, certified, from, now, location[:id])
+      end
+      result
+    end
+
+    def values_for(evaluator, certified, from, to, location_id)
+      certified.each_with_object({}) do |kpi, result|
+        result[kpi[:id]] = evaluator.value(
+          kpi,
+          from: from,
+          to: to,
+          location_id: location_id
+        )
+      end
+    end
+  end
+end