cubest 0.1.0

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "cubest",
+  "version": "0.1.0",
+  "description": "Single-pass OLAP aggregator over any text stream (code, logs, CSV, JSONL, XML, SDD artefacts) → 12 output formats including ECharts HTML. Token-cheap alternative to grep+cat chains for LLM agents. npm wrapper — installs Python cubest via pipx or delegates to python3.",
+  "keywords": [
+    "olap", "aggregator", "cli", "log-analysis", "observability", "sre",
+    "devops", "echarts", "graphviz", "mermaid", "plantuml", "code-analysis",
+    "loc-counter", "seo-audit", "sitemap", "llm", "claude-code", "ai-agents",
+    "token-optimization", "regex", "nginx", "percentile"
+  ],
+  "author": "Sergey Baryshev",
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/BaryshevS/cubest#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BaryshevS/cubest.git"
+  },
+  "bugs": {
+    "url": "https://github.com/BaryshevS/cubest/issues"
+  },
+  "bin": {
+    "cubest": "bin/cubest.js"
+  },
+  "files": ["bin/", "cubest.py", "profiles/", "README.md", "LICENSE", "NOTICE"],
+  "engines": {
+    "node": ">=14"
+  }
+}

package/profiles/agents_inventory.yaml

@@ -0,0 +1,15 @@
+name: agents_inventory
+description: "Claude subagent catalog: model × name via frontmatter of agents/*.md"
+scan:
+  include: ["agents/**/*.md", "*/agents/*.md", "**/subagents/*.md"]
+dimensions: [model, name]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: preset
+    preset: md_frontmatter
+output:
+  format: flat
+  top_n: 60
+  max_lines: 120

package/profiles/api_routes.yaml

@@ -0,0 +1,20 @@
+name: api_routes
+description: "FastAPI/Flask/Django endpoints by HTTP method and file"
+scan:
+  include: ["*.py"]
+  exclude: ["*test*", "venv/*", ".venv/*", "__pycache__/*"]
+dimensions: [method, file]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: regex
+    pattern: |
+      @(?:app|router|blueprint)\.\s*
+      (?P<method>get|post|put|delete|patch|options|head)
+      \s*\(\s*['"](?P<route>[^'"]+)['"]
+    multiline: true
+    ignorecase: true
+output:
+  format: tree
+  top_n: 20

package/profiles/call_graph.yaml

@@ -0,0 +1,19 @@
+name: call_graph
+description: "Approximate call graph: caller → callee (Python-like indent scope). Emit as GraphViz DOT. For exact AST-level graph use tree-sitter tools (Graphify)."
+scan:
+  include: ["*.py"]
+  exclude: [".git/", "node_modules/", "venv/", ".venv/", "__pycache__/",
+            "*_test.py", "tests/", "test_*.py"]
+dimensions: [caller, callee]
+measures:
+  - name: calls
+    type: count
+extract:
+  - type: preset
+    preset: calls
+filters:
+  - "caller != callee"                   # drop trivial self-loops
+output:
+  format: dot
+  top_n: 200
+  min_count: 1

package/profiles/code_atlas.yaml

@@ -0,0 +1,33 @@
+name: code_atlas
+description: "Multi-language code map: language × file × function (top 10 languages by extension). Python funcs get nesting (parent, depth)."
+scan:
+  include:
+    - "*.py"
+    - "*.js"
+    - "*.ts"
+    - "*.jsx"
+    - "*.tsx"
+    - "*.go"
+    - "*.rs"
+    - "*.java"
+    - "*.rb"
+    - "*.php"
+    - "*.kt"
+    - "*.swift"
+    - "*.c"
+    - "*.cpp"
+    - "*.sh"
+  exclude: [".git/", "node_modules/", "vendor/", "dist/", "build/",
+            "target/", "__pycache__/", "*.test.*", "*_test.go", "*.spec.*"]
+dimensions: [lang, file, parent, name]
+measures:
+  - name: symbols
+    type: count
+extract:
+  - type: preset
+    preset: funcs
+output:
+  format: tree
+  top_n: 20
+  min_count: 1
+  max_lines: 150

package/profiles/code_stats.yaml

@@ -0,0 +1,15 @@
+name: code_stats
+description: "Classes and functions per file with average lines"
+scan:
+  include: ["*.py", "*.js", "*.ts", "*.go", "*.rs"]
+  exclude: ["node_modules/*", "vendor/*", "*_test.go"]
+dimensions: [kind, file]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: preset
+    preset: funcs
+output:
+  format: tree
+  min_count: 1

package/profiles/csv_analytics.yaml

@@ -0,0 +1,31 @@
+name: csv_analytics
+description: "Ad/analytics tabular exports (GA4/AdWords/Yandex Metrica/Facebook Ads CSV or TSV). Native CSV parser with header row auto-detect. Column names are lowercased + non-alnum → _."
+scan:
+  include: ["*.csv", "*.tsv", "*.csv.gz", "*.tsv.gz"]
+  # `csv` preset forces batch mode (needs whole file for header parsing).
+dimensions: [campaign, device]
+measures:
+  - name: rows
+    type: count
+  - name: impressions
+    type: sum
+    field: impressions
+  - name: clicks
+    type: sum
+    field: clicks
+  - name: cost
+    type: sum
+    field: cost
+  - name: cost_p95
+    type: p95
+    field: cost
+extract:
+  - type: preset
+    preset: csv        # use `tsv` for tab-separated; add `sep: ";"` for European CSV
+    # header: true     # default; can also pass explicit list: [campaign, device, ...]
+    # skip: 1          # extra rows to drop before header (e.g. GA4 report header)
+output:
+  format: flat
+  top_n: 30
+  min_count: 1
+  max_lines: 80

package/profiles/disk_usage.yaml

@@ -0,0 +1,29 @@
+name: disk_usage
+description: "Disk usage aggregation up to depth 2. Use dimensions=[path_1..path_N] to change the level. Optional content_match filter includes only files with matching contents."
+scan:
+  include: ["*"]
+  exclude: [".git/", "node_modules/", "__pycache__/", ".venv/", "venv/",
+            "dist/", "build/", "*.lock"]
+  # Uncomment to include only files whose content matches (regex):
+  # content_match: ["TODO", "@deprecated"]
+  # content_not:   ["generated"]
+  # content_scan_bytes: 65536   # scan only first 64KiB per file (fast pre-check)
+dimensions: [path_1, path_2]
+measures:
+  - name: files
+    type: count
+  - name: bytes
+    type: sum
+    field: size
+  - name: max_file
+    type: max
+    field: size
+extract:
+  - type: preset
+    preset: paths
+output:
+  format: flat
+  top_n: 20
+  min_count: 1
+  human_bytes: true
+  max_lines: 60

package/profiles/doc_structure.yaml

@@ -0,0 +1,16 @@
+name: doc_structure
+description: "Markdown headers grouped by file and level"
+scan:
+  include: ["*.md", "*.mdx"]
+  exclude: ["node_modules/*"]
+dimensions: [file, level]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: preset
+    preset: headers
+filters:
+  - "level <= 3"
+output:
+  format: tree

package/profiles/file_tree.yaml

@@ -0,0 +1,27 @@
+name: file_tree
+description: "File tree by top-level directory and extension (no content read)"
+scan:
+  include: ["*"]
+  exclude:
+    - ".git/"
+    - "node_modules/"
+    - "__pycache__/"
+    - ".venv/"
+    - "venv/"
+    - "dist/"
+    - "build/"
+    - "*.lock"
+    - "*.pyc"
+dimensions: [top, ext]
+measures:
+  - name: count
+    type: count
+  - name: bytes
+    type: sum
+    field: size
+extract:
+  - type: preset
+    preset: paths
+output:
+  format: tree
+  min_count: 1

package/profiles/frontend_geoip.yaml

@@ -0,0 +1,49 @@
+name: frontend_geoip
+description: "Frontend/CDN access log analysis with GeoIP: filter by country + UA + endpoint + status 200; group by file extension; show hits, p90(duration), avg(size). Expects log rows enriched with GeoIP fields (see README `tools/geoip_enrich.sh`). Usage example — mobile Chrome from US/CA for /pub/covers/*: `python cubest.py --profile frontend_geoip enriched.log.gz`"
+scan:
+  include: ["access*.log*", "*.enriched.log*", "cdn*.log*"]
+  stream: true
+dimensions: [ext]
+measures:
+  - name: hits
+    type: count
+  - name: p50_ms
+    type: p50
+    field: duration
+  - name: p90_ms
+    type: p90
+    field: duration
+  - name: p99_ms
+    type: p99
+    field: duration
+  - name: avg_size
+    type: avg
+    field: bytes
+  - name: bytes_total
+    type: sum
+    field: bytes
+extract:
+  # Expected enriched combined format (tab-separated tail with country code):
+  #   IP - - [datetime] "METHOD /path HTTP/x.y" status bytes "referer" "UA" duration COUNTRY REGION
+  # Adjust the regex to match your ingest schema. Country + region come from
+  # GeoIP enrichment step (see `tools/geoip_enrich.sh`).
+  - type: regex
+    multiline: false
+    pattern: >-
+      (?P<ip>\d+\.\d+\.\d+\.\d+)\s+\S+\s+\S+\s+\[(?P<datetime>[^\]]+)\]\s+"(?P<method>GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH)\s+(?P<path>/[^\s?"]+)(?:\?[^\s"]*)?\s+HTTP/[\d.]+"\s+(?P<status>\d{3})\s+(?P<bytes>\d+)\s+"[^"]*"\s+"(?P<ua>[^"]*)"\s+(?P<duration>[0-9.]+)\s+(?P<country>[A-Z]{2})(?:\s+(?P<region>[A-Z0-9\-]+))?
+  # Second rule extracts extension from path (works because iteration
+  # merges named groups into the same record when applied to same line).
+  - type: regex
+    multiline: false
+    pattern: '/pub/covers/[^\s?"]+\.(?P<ext>webp|jpg|png|avif|gif)'
+    ignorecase: true
+filters:
+  - "status == 200"
+  - "country == 'US' or country == 'CA'"
+  - "'Mobile' in ua and 'Chrome' in ua"
+  - "ext in ('webp', 'jpg', 'png', 'avif', 'gif')"
+output:
+  format: echarts
+  chart_type: bar
+  top_n: 20
+  title: "Mobile Chrome (US/CA) /pub/covers/* — 200 by ext"

package/profiles/git_log_activity.yaml

@@ -0,0 +1,29 @@
+name: git_log_activity
+description: "Author × month activity from `git log --numstat --format=...` output piped as a single file. Requires a specific pretty format."
+scan:
+  include: ["*.log", "*.txt"]
+  stream: true
+dimensions: [author, month]
+measures:
+  - name: commits
+    type: count
+  - name: added
+    type: sum
+    field: added
+  - name: removed
+    type: sum
+    field: removed
+extract:
+  # Feed with:
+  #   git log --numstat --format='COMMIT|%an|%ai' --since=6.months > /tmp/gitlog.txt
+  # then parse both a COMMIT header line and numstat lines.
+  - type: regex
+    multiline: false
+    pattern: '^COMMIT\|(?P<author>[^|]+)\|(?P<month>\d{4}-\d{2})'
+  - type: regex
+    multiline: false
+    pattern: '^(?P<added>\d+)\s+(?P<removed>\d+)\s+\S+'
+output:
+  format: md_table
+  top_n: 40
+  max_lines: 60

package/profiles/imports.yaml

@@ -0,0 +1,16 @@
+name: imports
+description: "Python imports grouped by module"
+scan:
+  include: ["*.py"]
+  exclude: ["*test*", "venv/*"]
+dimensions: [module, file]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: regex
+    pattern: '^(?:from|import)\s+(?P<module>[a-zA-Z_][a-zA-Z0-9_.]*)'
+  - type: regex
+    pattern: '^from\s+(?P<module>[a-zA-Z_][a-zA-Z0-9_.]*)\s+import'
+output:
+  format: compact

package/profiles/jsonl_events.yaml

@@ -0,0 +1,19 @@
+name: jsonl_events
+description: "JSON-lines event stream (app events, ClickHouse SELECT ... FORMAT JSONEachRow, Kafka dumps): event × source with counts"
+scan:
+  include: ["*.jsonl", "*.jsonl.gz", "*.ndjson", "*.ndjson.gz"]
+  stream: true
+dimensions: [event, source]
+measures:
+  - name: rows
+    type: count
+extract:
+  - type: regex
+    multiline: false
+    pattern: >-
+      "event"\s*:\s*"(?P<event>[^"]+)"[^\n]*?"source"\s*:\s*"(?P<source>[^"]+)"
+output:
+  format: flat
+  top_n: 30
+  min_count: 1
+  max_lines: 80

package/profiles/k8s_resources.yaml

@@ -0,0 +1,25 @@
+name: k8s_resources
+description: "Kubernetes manifests inventory: kind × namespace × name across a helm chart / kustomize / raw yaml tree."
+scan:
+  include: ["*.yaml", "*.yml"]
+  exclude: [".git/", "node_modules/"]
+  content_match: ['^apiVersion:\s']
+  stream: true
+dimensions: [kind, namespace, name]
+measures:
+  - name: docs
+    type: count
+extract:
+  - type: regex
+    multiline: false
+    pattern: '^kind:\s*(?P<kind>\S+)'
+  - type: regex
+    multiline: false
+    pattern: '^\s{2}namespace:\s*(?P<namespace>\S+)'
+  - type: regex
+    multiline: false
+    pattern: '^\s{2}name:\s*(?P<name>\S+)'
+output:
+  format: flat
+  top_n: 30
+  max_lines: 80

package/profiles/loc_counter.yaml

@@ -0,0 +1,51 @@
+name: loc_counter
+description: "Lines of code by language and top-level directory. Alternative to scc/tokei/cloc for quick estimates. Splits code vs comments vs blanks."
+scan:
+  include:
+    - "*.py"
+    - "*.js"
+    - "*.ts"
+    - "*.jsx"
+    - "*.tsx"
+    - "*.go"
+    - "*.rs"
+    - "*.java"
+    - "*.rb"
+    - "*.php"
+    - "*.kt"
+    - "*.swift"
+    - "*.c"
+    - "*.h"
+    - "*.cpp"
+    - "*.hpp"
+    - "*.cs"
+    - "*.sh"
+    - "*.sql"
+    - "*.yaml"
+    - "*.yml"
+    - "*.json"
+    - "*.md"
+  exclude: [".git/", "node_modules/", "vendor/", "dist/", "build/",
+            "target/", "*.min.*", "*.lock", "__pycache__/"]
+dimensions: [ext, top]
+measures:
+  - name: lines
+    type: count
+  - name: blanks
+    type: sum
+    field: blank
+  - name: comments
+    type: sum
+    field: comment
+  - name: bytes
+    type: sum
+    field: length
+extract:
+  - type: preset
+    preset: lines
+output:
+  format: flat
+  top_n: 50
+  min_count: 1
+  human_bytes: true
+  max_lines: 80

package/profiles/mr_impact.yaml

@@ -0,0 +1,19 @@
+name: mr_impact
+description: "MR/PR impact: distribution of changed files across top-level dir × extension with size. Feed changed files via --files-from -."
+scan:
+  include: ["*"]
+dimensions: [top, ext]
+measures:
+  - name: files
+    type: count
+  - name: bytes
+    type: sum
+    field: size
+extract:
+  - type: preset
+    preset: paths
+output:
+  format: flat
+  top_n: 20
+  human_bytes: true
+  max_lines: 40

package/profiles/nginx_access.yaml

@@ -0,0 +1,33 @@
+name: nginx_access
+description: "Nginx combined access log: top URL sections × HTTP status with avg duration and total bytes"
+scan:
+  include: ["access*.log*", "*.access.log*"]
+  stream: true
+dimensions: [status, method, path_root]
+measures:
+  - name: hits
+    type: count
+  - name: avg_dur
+    type: avg
+    field: duration
+  - name: bytes_total
+    type: sum
+    field: bytes
+  - name: max_dur
+    type: max
+    field: duration
+extract:
+  # Combined log format + $request_time on tail; tolerates leading
+  # "[host date]" decorator (some ELB variants).
+  - type: regex
+    multiline: false
+    pattern: >-
+      (?:\[[^\]]+\]\s+)?(?P<ip>\d+\.\d+\.\d+\.\d+)\s+\S+\s+\S+\s+\[(?P<datetime>[^\]]+)\]\s+"?(?P<method>GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH)\s+/(?P<path_root>[^/?"\s]*)(?:/[^\s"?]*)?(?:\?[^\s"]*)?\s+HTTP/[\d.]+"?\s+(?P<status>\d{3})\s+(?P<bytes>\d+|-)(?:\s+"[^"]*"\s+"[^"]*")?(?:\s+"[^"]*")?(?:\s+(?P<duration>[0-9.]+|-))?
+filters:
+  - "bytes != '-'"
+output:
+  format: flat
+  top_n: 20
+  min_count: 5
+  human_bytes: true
+  max_lines: 60

package/profiles/nginx_cdn_covers.yaml

@@ -0,0 +1,32 @@
+name: nginx_cdn_covers
+description: "CDN TSV log: /pub/c/cover_SIZE/ID.FORMAT — size × format × device with avg upstream time (port of cdn_stat.pl, no GeoIP)"
+scan:
+  include: ["*.tsv*", "*.txt*"]
+  stream: true
+dimensions: [size, format, device]
+measures:
+  - name: hits
+    type: count
+  - name: avg_dur
+    type: avg
+    field: duration
+  - name: avg_upstream
+    type: avg
+    field: upstream_response_time
+extract:
+  # Assumes tab-separated columns; picks the request cell via anchor,
+  # duration cell via the standard nginx $request_time float. `device` is
+  # derived below from the useragent match on 'Mobile'.
+  - type: regex
+    multiline: false
+    ignorecase: true
+    pattern: >-
+      GET\s+/pub/c/cover_(?P<size>\d+)/(?P<artid>\d+)\.(?P<format>webp|jpg)[^\t]*\t(?P<status>\d{3})\t(?P<duration>[0-9.]+)\t(?P<upstream_response_time>[0-9.]+|-)\t(?P<ua>[^\t]*)(?P<device>Mobile)?
+filters:
+  - "status == 200 or status == 304"
+  - "size == 330 or size == 415"
+output:
+  format: flat
+  top_n: 25
+  min_count: 3
+  max_lines: 80

package/profiles/openapi_endpoints.yaml

@@ -0,0 +1,19 @@
+name: openapi_endpoints
+description: "OpenAPI/Swagger spec inventory: method × path (parses both YAML and JSON specs). Detects operationId if present."
+scan:
+  include: ["*openapi*.yaml", "*openapi*.yml", "*openapi*.json",
+            "*swagger*.yaml", "*swagger*.yml", "*swagger*.json"]
+dimensions: [method, path]
+measures:
+  - name: ops
+    type: count
+extract:
+  # match `  /path:` followed by `    get:` etc. within an indented paths: block.
+  - type: regex
+    multiline: true
+    pattern: '^\s{2,4}(?P<path>/[^\s:]+):\s*\n(?:^\s{4,6}(?P<method>get|post|put|delete|patch|options|head):)'
+    ignorecase: true
+output:
+  format: flat
+  top_n: 40
+  max_lines: 100

package/profiles/react_components.yaml

@@ -0,0 +1,15 @@
+name: react_components
+description: "React components by type (function/class) and file"
+scan:
+  include: ["*.jsx", "*.tsx", "*.js", "*.ts"]
+  exclude: ["node_modules/*", "*.test.*", "*.spec.*"]
+dimensions: [type, file]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: regex
+    pattern: '(?P<type>function|const|class)\s+(?P<name>[A-Z][a-zA-Z0-9]*)\s*[=(]'
+output:
+  format: tree
+  min_count: 1

package/profiles/sdd_checklist.yaml

@@ -0,0 +1,17 @@
+name: sdd_checklist
+description: "Progress across Markdown checklists (Phase 1 gates, capability preflight, acceptance tests): file × done/todo counts"
+scan:
+  include: ["*.md", "*.mdx"]
+  exclude: [".git/", "node_modules/"]
+dimensions: [state, file]
+measures:
+  - name: items
+    type: count
+extract:
+  - type: preset
+    preset: md_checklist
+output:
+  format: tree
+  top_n: 25
+  min_count: 1
+  max_lines: 80

package/profiles/sdd_specs.yaml

@@ -0,0 +1,19 @@
+name: sdd_specs
+description: "SDD/spec-artefact inventory via YAML frontmatter: type × status per file (skills, agents, PRDs, ADRs)"
+scan:
+  include: ["*.md", "*.mdx"]
+  exclude: [".git/", "node_modules/", "dist/"]
+dimensions: [type, status, name]
+measures:
+  - name: count
+    type: count
+extract:
+  - type: preset
+    preset: md_frontmatter
+filters:
+  # only files that had a frontmatter block
+  - "type != '---' or status != '---'"
+output:
+  format: flat
+  top_n: 40
+  max_lines: 100

package/profiles/seo_audit.yaml

@@ -0,0 +1,31 @@
+name: seo_audit
+description: "SEO audit of a folder of HTML pages: title/description length distribution, missing/duplicate H1, canonical presence, JSON-LD schema. Feed with a site crawl or static export."
+scan:
+  include: ["*.html", "*.htm", "*.xhtml"]
+  exclude: [".git/", "node_modules/", "dist/", "build/"]
+dimensions: [lang, twitter_card]
+measures:
+  - name: pages
+    type: count
+  - name: title_avg
+    type: avg
+    field: title_len
+  - name: title_max
+    type: max
+    field: title_len
+  - name: desc_avg
+    type: avg
+    field: desc_len
+  - name: h1_total
+    type: sum
+    field: h1_count
+  - name: schema_pages
+    type: sum
+    field: has_schema
+extract:
+  - type: preset
+    preset: html_meta
+output:
+  format: md_table
+  top_n: 40
+  max_lines: 60

package/profiles/seo_semantic_tree.yaml

@@ -0,0 +1,17 @@
+name: seo_semantic_tree
+description: "Semantic heading tree per page: file × H-level × title. Use to spot orphan H2/H3, missing H1s, unbalanced structure across a site. Render as sunburst/tree/treemap in ECharts."
+scan:
+  include: ["*.html", "*.htm", "*.xhtml"]
+  exclude: [".git/", "node_modules/"]
+dimensions: [file, level, title]
+measures:
+  - name: occurrences
+    type: count
+extract:
+  - type: preset
+    preset: html_headings
+output:
+  format: echarts
+  chart_type: sunburst
+  top_n: 40
+  title: "Semantic heading tree"

package/profiles/sitemap_map.yaml

@@ -0,0 +1,20 @@
+name: sitemap_map
+description: "Sitemap.xml URL inventory: host × section (up to 3 URL segments) with priority/lastmod distribution. Supports both `<urlset>` (regular sitemaps) and `<sitemapindex>` (index files)."
+scan:
+  include: ["sitemap*.xml", "*sitemap*.xml", "*.sitemap.xml"]
+dimensions: [host, section_1, section_2]
+measures:
+  - name: urls
+    type: count
+  - name: priority_avg
+    type: avg
+    field: priority
+extract:
+  - type: preset
+    preset: sitemap
+output:
+  format: echarts
+  chart_type: treemap
+  top_n: 100
+  min_count: 1
+  title: "Site URL taxonomy"