cli-community-intelligence 0.1.18 → 0.1.20

package/dist/cli.js

@@ -5680,6 +5680,7 @@ var queryPosts = (db, { source, channel, text, author, since, until, minScore, l
   const sql = `SELECT * FROM posts ${where} ORDER BY createdAt DESC ${limitClause}`;
   return db.prepare(sql).all(...params);
 };
+var querySql = (db, sql) => db.prepare(sql).all();
 var CorpusDb = {
   createDb,
   persistPost,
@@ -5692,6 +5693,7 @@ var CorpusDb = {
   queryPostBySourceId,
   queryRevisitQueue,
   queryScrapeRuns,
+  querySql,
   updateQueueEntry,
   removeFromQueue,
   countChildPosts
@@ -5819,11 +5821,11 @@ var T2 = {
     return cutoff.toISOString();
   },
   // Determine the cutoff date for a subreddit based on incremental/backfill settings
-  // @sig toCutoffForSubreddit :: (String, Boolean, Number?, Function?) -> String?
+  // WHY: default to 30 days so bare `scrape reddit` has predictable scope
+  // @sig toCutoffForSubreddit :: (String, Boolean, Number?, Function?) -> String
   toCutoffForSubreddit: (subreddit, incremental, backfillDays, getLastCreatedAt) => {
     if (incremental && getLastCreatedAt) return getLastCreatedAt("reddit", subreddit);
-    if (backfillDays) return T2.toCutoffDate(backfillDays);
-    return void 0;
+    return T2.toCutoffDate(backfillDays || DEFAULT_BACKFILL_DAYS);
   },
   // Build a scrape configuration object for a single subreddit
   // @sig toScrapeConfig :: (Object, String, Boolean, Number?, Function?) -> Object
@@ -5942,6 +5944,7 @@ var POSTS_PER_PAGE_UNAUTH = 25;
 var RATE_LIMIT_DELAY_AUTH = 700;
 var RATE_LIMIT_DELAY_UNAUTH2 = RedditApi.RATE_LIMIT_DELAY_UNAUTH;
 var MAX_POSTS_PER_SUBREDDIT = 1e3;
+var DEFAULT_BACKFILL_DAYS = 30;
 var gatherReddit = async ({ subreddits, backfillDays, incremental, getLastCreatedAt }) => {
   const subredditList = RedditApi.toSubredditList(subreddits);
   const auth = await E.requestToken();
@@ -6346,6 +6349,14 @@ var E3 = {
     E3.emitStatus(CorpusDb.queryStatus(db));
     db.close();
   },
+  // Handle sql command — run a read-only SQL query against the corpus
+  // @sig handleSql :: Object -> void
+  handleSql: (argv) => {
+    const db = CorpusDb.createDb(argv.db);
+    const rows = CorpusDb.querySql(db, argv.sql);
+    console.log(JSON.stringify(rows, void 0, 2));
+    db.close();
+  },
   // Handle query command — output matching posts as formatted text
   // @sig handleQuery :: Object -> void
   handleQuery: (argv) => {
@@ -6416,8 +6427,9 @@ ${posts.length} post(s) found
       () => posts.forEach((post) => E3.persistPostRevisitEntry(db, post, moreCounts[post.sourceId] || 0))
     );
     enqueueAll();
+    const dupes = allPosts.length - inserted;
     console.log(`
-Stored ${inserted} new posts (${allPosts.length - inserted} duplicates skipped)`);
+Stored ${posts.length} posts and ${comments.length} comments (${dupes} duplicates skipped)`);
     console.log(`Queued ${posts.length} posts for revisit`);
     db.close();
   },
@@ -6477,7 +6489,7 @@ Stored ${inserted} new posts (${posts.length - inserted} duplicates skipped)`);
   // Run the CLI — parse args and dispatch to handlers
   // @sig runCli :: () -> void
   runCli: () => {
-    const { handleExport, handleFatalError, handleQuery, handleRevisit, handleScrape, handleStatus } = E3;
+    const { handleExport, handleFatalError, handleQuery, handleRevisit, handleScrape, handleSql, handleStatus } = E3;
     const { buildFilterOptions, buildRevisitOptions, buildScrapeOptions } = F3;
     try {
       yargs_default(hideBin(process.argv)).usage("Usage: community-intelligence <command> [options]").option("db", {
@@ -6489,7 +6501,12 @@ Stored ${inserted} new posts (${posts.length - inserted} duplicates skipped)`);
         "Re-fetch queued posts for new comments",
         buildRevisitOptions,
         handleRevisit
-      ).command("query", "Search the corpus", buildFilterOptions, handleQuery).command("export", "Export matching posts as JSON to stdout", buildFilterOptions, handleExport).demandCommand(1, "Please specify a command").help().alias("help", "h").version().alias("version", "v").strict().parse();
+      ).command("query", "Search the corpus", buildFilterOptions, handleQuery).command("export", "Export matching posts as JSON to stdout", buildFilterOptions, handleExport).command(
+        "sql <sql>",
+        "Run a read-only SQL query against the corpus",
+        (y) => y.positional("sql", { describe: "SQL query to execute", type: "string" }),
+        handleSql
+      ).demandCommand(1, "Please specify a command").help().alias("help", "h").version().alias("version", "v").strict().parse();
     } catch (error) {
       handleFatalError(error);
     }

package/dist/index.js

@@ -823,6 +823,7 @@ var queryPosts = (db, { source, channel, text, author, since, until, minScore, l
   const sql = `SELECT * FROM posts ${where} ORDER BY createdAt DESC ${limitClause}`;
   return db.prepare(sql).all(...params);
 };
+var querySql = (db, sql) => db.prepare(sql).all();
 var CorpusDb = {
   createDb,
   persistPost,
@@ -835,6 +836,7 @@ var CorpusDb = {
   queryPostBySourceId,
   queryRevisitQueue,
   queryScrapeRuns,
+  querySql,
   updateQueueEntry,
   removeFromQueue,
   countChildPosts

package/openclaw-extension.js

@@ -75,10 +75,15 @@ const queryFilterParams = Type.Object({
 //
 // ---------------------------------------------------------------------------------------------------------------------
 
+// COMPLEXITY: export-structure — OpenClaw's definePluginEntry requires export default
+// @sig OpenClawExtension :: OpenClawPluginDefinition
 export default definePluginEntry({
     id: 'cli-community-intelligence',
     name: 'Community Intelligence',
     description: 'Scrape and query posts from online communities (Reddit, YouTube)',
+
+    // Register community-intelligence tools with OpenClaw's plugin API
+    // @sig register :: OpenClawPluginApi -> void
     register(api) {
         api.registerTool({
             name: 'community-intelligence-scrape',
@@ -104,7 +109,8 @@ export default definePluginEntry({
         api.registerTool({
             name: 'community-intelligence-query',
             description:
-                'Search the scraped community posts corpus. Filter by source, channel, text, author, date range, or score.',
+                'Search the scraped community posts corpus. ' +
+                'Filter by source, channel, text, author, date range, or score.',
             parameters: queryFilterParams,
             execute: F.buildExecutor('query'),
         })
@@ -122,5 +128,16 @@ export default definePluginEntry({
             parameters: queryFilterParams,
             execute: F.buildExecutor('export'),
         })
+
+        api.registerTool({
+            name: 'community-intelligence-sql',
+            description:
+                'Run a read-only SQL query against the community intelligence corpus. ' +
+                'The database has a "posts" table with columns: source, sourceId, channel, ' +
+                'author, title, body, parentId, url, score, createdAt, postType, scrapedAt. ' +
+                'Use standard SQLite syntax for aggregations, grouping, etc.',
+            parameters: Type.Object({ sql: Type.String({ description: 'SQL query to execute (read-only)' }) }),
+            execute: F.buildExecutor('sql'),
+        })
     },
 })

package/openclaw.plugin.json

@@ -2,13 +2,14 @@
     "id": "cli-community-intelligence",
     "name": "Community Intelligence",
     "description": "Scrape and query posts from online communities (Reddit, YouTube)",
-    "version": "0.1.18",
+    "version": "0.1.20",
     "contracts": {
         "tools": [
             "community-intelligence-scrape",
             "community-intelligence-query",
             "community-intelligence-status",
-            "community-intelligence-export"
+            "community-intelligence-export",
+            "community-intelligence-sql"
         ]
     },
     "configSchema": {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "cli-community-intelligence",
-    "version": "0.1.18",
+    "version": "0.1.20",
     "description": "Community intelligence scraper for construction industry market research",
     "type": "module",
     "main": "dist/index.js",
@@ -21,7 +21,11 @@
         }
     },
     "dependencies": {
-        "better-sqlite3": "^11.0.0",
         "@sinclair/typebox": "0.34.49"
-    }
+    },
+    "bundledDependencies": [
+        "better-sqlite3",
+        "bindings",
+        "file-uri-to-path"
+    ]
 }