cli-community-intelligence 0.1.5

package/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "cli-community-intelligence",
+    "version": "0.1.5",
+    "description": "Community intelligence scraper for construction industry market research",
+    "type": "module",
+    "main": "dist/index.js",
+    "bin": {
+        "community-intelligence": "dist/cli.js"
+    },
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/graffio/graffio-monorepo.git"
+    },
+    "dependencies": {
+        "better-sqlite3": "^8.7.0"
+    }
+}

package/schema.sql

@@ -0,0 +1,66 @@
+-- =====================================================
+-- Community Intelligence Corpus Schema
+-- =====================================================
+--
+-- Stores scraped posts and comments from online communities.
+-- Column names use camelCase to match JavaScript conventions.
+--
+
+CREATE TABLE IF NOT EXISTS posts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,                     -- 'reddit', 'youtube', 'forum'
+    sourceId TEXT NOT NULL,                   -- Original ID from source platform
+    channel TEXT NOT NULL,                    -- Subreddit, YouTube channel, forum name
+    author TEXT,                              -- Username (may be null if deleted)
+    title TEXT,                               -- Post title (null for comments)
+    body TEXT NOT NULL,                       -- Post or comment text content
+    parentId TEXT,                            -- Parent post sourceId (for comments)
+    url TEXT,                                 -- Permalink
+    score INTEGER,                            -- Upvotes, likes, etc.
+    createdAt TEXT NOT NULL,                  -- ISO 8601 timestamp from source
+    scrapedAt TEXT NOT NULL DEFAULT (datetime('now')), -- When we scraped it
+    postType TEXT NOT NULL CHECK (postType IN ('post', 'comment'))
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_posts_source_sourceId ON posts(source, sourceId);
+CREATE INDEX IF NOT EXISTS idx_posts_channel ON posts(channel);
+CREATE INDEX IF NOT EXISTS idx_posts_createdAt ON posts(createdAt);
+CREATE INDEX IF NOT EXISTS idx_posts_source_channel_createdAt ON posts(source, channel, createdAt DESC);
+
+-- =====================================================
+-- Scrape Runs — logs each scrape operation for analytics
+-- =====================================================
+
+CREATE TABLE IF NOT EXISTS scrape_runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,                     -- 'reddit', 'youtube', 'forum'
+    channel TEXT NOT NULL,                    -- Subreddit, YouTube channel, forum name
+    startedAt TEXT NOT NULL,                  -- ISO 8601 timestamp
+    completedAt TEXT NOT NULL,                -- ISO 8601 timestamp
+    postsFound INTEGER NOT NULL DEFAULT 0,
+    postsInserted INTEGER NOT NULL DEFAULT 0,
+    commentsFound INTEGER NOT NULL DEFAULT 0,
+    commentsInserted INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_scrape_runs_source_channel ON scrape_runs(source, channel);
+
+-- =====================================================
+-- Revisit Queue — tracks posts needing follow-up scrapes
+-- =====================================================
+--
+-- One entry per post. reason is the most specific: has_more > high_score > recent.
+-- Exit conditions (OR): post age > threshold OR unchangedCount > threshold.
+--
+
+CREATE TABLE IF NOT EXISTS revisit_queue (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,                     -- 'reddit', 'youtube', 'forum'
+    sourceId TEXT NOT NULL,                   -- Post sourceId from the posts table
+    reason TEXT NOT NULL CHECK (reason IN ('recent', 'high_score', 'has_more')),
+    moreCount INTEGER NOT NULL DEFAULT 0,    -- Count of "more" placeholder objects in comment tree
+    lastCheckedAt TEXT,                       -- ISO 8601 timestamp of last revisit
+    unchangedCount INTEGER NOT NULL DEFAULT 0 -- Consecutive revisits with zero new comments
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_revisit_queue_source_sourceId ON revisit_queue(source, sourceId);