amaprice 1.0.9 → 1.0.11

package/README.md

@@ -177,6 +177,13 @@ Environment variables used by the npm package:
 | `SYNC_INTERVAL_MINUTES` | `5` | `src/worker.js` | Worker loop interval |
 | `SYNC_LIMIT` | `20` | `src/worker.js`, `amaprice sync --limit` | Max due products per run |
 | `SYNC_RUN_ONCE` | `0` | `src/worker.js` | Set `1` for single run and exit |
+| `VISION_FALLBACK_ENABLED` | `0` | `src/extractors/pipeline.js` | Enable screenshot + vision fallback when HTML/JSON extraction fails |
+| `OPENROUTER_API_KEY` | none | `src/extractors/vision.js` | Preferred vision provider key |
+| `VISION_MODEL` | `google/gemini-3-flash-preview` | `src/extractors/vision.js` | OpenRouter model ID for vision extraction |
+| `VISION_PROVIDER` | auto | `src/extractors/vision.js` | Optional force value: `openrouter` or `openai` |
+| `OPENROUTER_HTTP_REFERER` | none | `src/extractors/vision.js` | Optional OpenRouter attribution header |
+| `OPENROUTER_TITLE` | none | `src/extractors/vision.js` | Optional OpenRouter attribution header |
+| `OPENAI_API_KEY` | none | `src/extractors/vision.js` | Legacy fallback if `OPENROUTER_API_KEY` is unset |
 
 For production background workers, prefer the Supabase **service role key**.
 
@@ -192,6 +199,9 @@ Steps:
 3. Optional env vars:
    - `SYNC_INTERVAL_MINUTES=5`
    - `SYNC_LIMIT=20`
+   - `VISION_FALLBACK_ENABLED=1`
+   - `OPENROUTER_API_KEY=<your-openrouter-key>`
+   - `VISION_MODEL=google/gemini-3-flash-preview`
 4. Ensure builder is Dockerfile (root `Dockerfile`).
 5. Deploy.
 6. Confirm logs show `[worker] processed=...`.

package/bin/cli.js

@@ -2,7 +2,19 @@
 
 const { program } = require('commander');
 const pkg = require('../package.json');
-const KNOWN_COMMANDS = new Set(['price', 'track', 'history', 'list', 'sync', 'tier', 'help']);
+const KNOWN_COMMANDS = new Set([
+  'price',
+  'track',
+  'history',
+  'list',
+  'sync',
+  'tier',
+  'subscribe',
+  'unsubscribe',
+  'subscriptions',
+  'collector',
+  'help',
+]);
 
 const userArgs = process.argv.slice(2);
 if (userArgs.length > 0) {
@@ -25,5 +37,9 @@ require('../src/commands/history')(program);
 require('../src/commands/list')(program);
 require('../src/commands/sync')(program);
 require('../src/commands/tier')(program);
+require('../src/commands/subscribe')(program);
+require('../src/commands/unsubscribe')(program);
+require('../src/commands/subscriptions')(program);
+require('../src/commands/collector')(program);
 
 program.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amaprice",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "CLI tool to scrape and track Amazon product prices",
   "main": "src/scraper.js",
   "type": "commonjs",
@@ -11,6 +11,9 @@
     "sync": "node bin/cli.js sync",
     "worker": "node src/worker.js",
     "worker:once": "SYNC_RUN_ONCE=1 node src/worker.js",
+    "collector:start": "node bin/cli.js collector start",
+    "collector:once": "node bin/cli.js collector run-once",
+    "analyze:scrape": "node scripts/analyze-scrape-attempts.js",
     "test": "node --test",
     "postinstall": "npx --yes playwright install chromium"
   },

package/src/collector/client.js

@@ -0,0 +1,68 @@
+const { runOrchestratedSync } = require('../orchestrator/runner');
+const { readCollectorState } = require('./state');
+const { heartbeatCollector } = require('../db');
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function ensureCollectorState() {
+  const state = await readCollectorState();
+  if (!state || !state.collectorId) {
+    throw new Error('Collector is not enabled. Run `amaprice collector enable` first.');
+  }
+  return state;
+}
+
+async function runCollectorOnce({ limit = 5 } = {}) {
+  const state = await ensureCollectorState();
+  await heartbeatCollector({
+    collectorId: state.collectorId,
+    status: state.status === 'paused' ? 'paused' : 'active',
+    capabilities: state.capabilities || { html_json: true, vision: true },
+  }).catch(() => {});
+
+  if (state.status === 'paused') {
+    return {
+      processed: 0,
+      success: 0,
+      failed: 0,
+      items: [],
+      paused: true,
+    };
+  }
+
+  return runOrchestratedSync({
+    limit,
+    collectorId: state.collectorId,
+    executor: 'collector',
+    routeHint: 'collector_first',
+    allowVision: true,
+    allowRailwayDomFallback: true,
+  });
+}
+
+async function runCollectorLoop({ limit = 5, pollSeconds = 20 } = {}) {
+  const safePollMs = Math.max(5, Number(pollSeconds) || 20) * 1000;
+
+  while (true) {
+    const started = Date.now();
+    const report = await runCollectorOnce({ limit });
+    const elapsed = Date.now() - started;
+    const waitMs = Math.max(0, safePollMs - elapsed);
+
+    if (report.paused) {
+      await sleep(safePollMs);
+      continue;
+    }
+
+    if (waitMs > 0) {
+      await sleep(waitMs);
+    }
+  }
+}
+
+module.exports = {
+  runCollectorOnce,
+  runCollectorLoop,
+};

package/src/collector/state.js

@@ -0,0 +1,50 @@
+const fs = require('fs/promises');
+const path = require('path');
+const os = require('os');
+
+function getStateDir() {
+  return process.env.AMAPRICE_STATE_DIR || path.join(os.homedir(), '.amaprice');
+}
+
+function getCollectorStatePath() {
+  return path.join(getStateDir(), 'collector.json');
+}
+
+async function readCollectorState() {
+  const target = getCollectorStatePath();
+  try {
+    const raw = await fs.readFile(target, 'utf8');
+    return JSON.parse(raw);
+  } catch (err) {
+    if (err && err.code === 'ENOENT') {
+      return null;
+    }
+    throw err;
+  }
+}
+
+async function writeCollectorState(state) {
+  const dir = getStateDir();
+  await fs.mkdir(dir, { recursive: true });
+  const target = getCollectorStatePath();
+  await fs.writeFile(target, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
+  return target;
+}
+
+async function clearCollectorState() {
+  const target = getCollectorStatePath();
+  try {
+    await fs.unlink(target);
+  } catch (err) {
+    if (!err || err.code !== 'ENOENT') {
+      throw err;
+    }
+  }
+}
+
+module.exports = {
+  getCollectorStatePath,
+  readCollectorState,
+  writeCollectorState,
+  clearCollectorState,
+};

package/src/commands/collector.js

@@ -0,0 +1,187 @@
+const os = require('os');
+const { getUserId } = require('../user-context');
+const {
+  upsertCollector,
+  getCollectorById,
+  heartbeatCollector,
+} = require('../db');
+const {
+  readCollectorState,
+  writeCollectorState,
+  clearCollectorState,
+  getCollectorStatePath,
+} = require('../collector/state');
+const {
+  runCollectorOnce,
+  runCollectorLoop,
+} = require('../collector/client');
+
+function getDefaultCollectorName() {
+  return `${os.hostname()}-collector`;
+}
+
+module.exports = function (program) {
+  program
+    .command('collector <action>')
+    .description('Manage local collector daemon (enable|disable|status|pause|resume|run-once|start)')
+    .option('--name <name>', 'Collector name override')
+    .option('--limit <n>', 'Max jobs per loop/once run', '5')
+    .option('--poll-seconds <n>', 'Polling interval for start loop', '20')
+    .option('--json', 'Output as JSON')
+    .action(async (action, opts) => {
+      const normalizedAction = String(action || '').trim().toLowerCase();
+      const userId = getUserId();
+      const limit = Math.max(1, Number(opts.limit) || 5);
+      const pollSeconds = Math.max(5, Number(opts.pollSeconds) || 20);
+
+      try {
+        if (normalizedAction === 'enable') {
+          const existing = await readCollectorState();
+          const collector = await upsertCollector({
+            collectorId: existing?.collectorId || null,
+            userId,
+            name: opts.name || existing?.name || getDefaultCollectorName(),
+            kind: 'cli',
+            status: 'active',
+            capabilities: {
+              html_json: true,
+              vision: true,
+              railway_dom: true,
+            },
+            metadata: {
+              platform: process.platform,
+              node: process.version,
+            },
+          });
+
+          const nextState = {
+            collectorId: collector.id,
+            userId,
+            name: collector.name,
+            status: 'active',
+            capabilities: collector.capabilities,
+            enabledAt: new Date().toISOString(),
+          };
+          const statePath = await writeCollectorState(nextState);
+
+          if (opts.json) {
+            console.log(JSON.stringify({
+              enabled: true,
+              collectorId: collector.id,
+              statePath,
+            }));
+            return;
+          }
+
+          console.log(`Collector enabled: ${collector.id}`);
+          console.log(`State file:        ${statePath}`);
+          return;
+        }
+
+        if (normalizedAction === 'disable') {
+          const existing = await readCollectorState();
+          if (existing?.collectorId) {
+            await heartbeatCollector({
+              collectorId: existing.collectorId,
+              status: 'revoked',
+            }).catch(() => {});
+          }
+          await clearCollectorState();
+
+          if (opts.json) {
+            console.log(JSON.stringify({ enabled: false }));
+            return;
+          }
+
+          console.log('Collector disabled.');
+          return;
+        }
+
+        if (normalizedAction === 'status') {
+          const state = await readCollectorState();
+          const remote = state?.collectorId
+            ? await getCollectorById(state.collectorId).catch(() => null)
+            : null;
+
+          if (opts.json) {
+            console.log(JSON.stringify({
+              statePath: getCollectorStatePath(),
+              local: state,
+              remote,
+            }));
+            return;
+          }
+
+          if (!state) {
+            console.log('Collector is not enabled.');
+            console.log(`State file: ${getCollectorStatePath()}`);
+            return;
+          }
+
+          console.log(`Collector ID: ${state.collectorId}`);
+          console.log(`Status:       ${state.status || 'active'}`);
+          console.log(`State file:   ${getCollectorStatePath()}`);
+          if (remote) {
+            console.log(`Last seen:    ${remote.last_seen_at || 'never'}`);
+            console.log(`Remote:       ${remote.status}`);
+          }
+          return;
+        }
+
+        if (normalizedAction === 'pause' || normalizedAction === 'resume') {
+          const state = await readCollectorState();
+          if (!state?.collectorId) {
+            throw new Error('Collector is not enabled. Run `amaprice collector enable` first.');
+          }
+
+          const status = normalizedAction === 'pause' ? 'paused' : 'active';
+          await heartbeatCollector({ collectorId: state.collectorId, status });
+          await writeCollectorState({
+            ...state,
+            status,
+            updatedAt: new Date().toISOString(),
+          });
+
+          if (opts.json) {
+            console.log(JSON.stringify({ collectorId: state.collectorId, status }));
+            return;
+          }
+
+          console.log(`Collector ${status}.`);
+          return;
+        }
+
+        if (normalizedAction === 'run-once') {
+          const report = await runCollectorOnce({ limit });
+          if (opts.json) {
+            console.log(JSON.stringify(report));
+            return;
+          }
+
+          if (report.paused) {
+            console.log('Collector is paused.');
+            return;
+          }
+
+          console.log(`Processed: ${report.processed}`);
+          console.log(`Success:   ${report.success}`);
+          console.log(`Failed:    ${report.failed}`);
+          return;
+        }
+
+        if (normalizedAction === 'start') {
+          if (!opts.json) {
+            console.log(`[collector] starting limit=${limit} poll_seconds=${pollSeconds}`);
+          }
+          await runCollectorLoop({ limit, pollSeconds });
+          return;
+        }
+
+        console.error('Unknown collector action. Use: enable, disable, status, pause, resume, run-once, start.');
+        process.exit(1);
+      } catch (err) {
+        console.error(`Error: ${err.message}`);
+        process.exit(1);
+      }
+    });
+};

package/src/commands/list.js

@@ -1,13 +1,60 @@
-const { listProducts } = require('../db');
+const { listProducts, getUserSubscriptions } = require('../db');
 const { formatPrice } = require('../format');
+const { getUserId } = require('../user-context');
 
 module.exports = function (program) {
   program
     .command('list')
-    .description('Show all tracked products with latest price')
+    .description('Show tracked products (subscriptions by default)')
+    .option('--global', 'Show global tracked products instead of current user subscriptions')
+    .option('--all', 'Include inactive subscriptions (subscriptions mode only)')
     .option('--json', 'Output as JSON')
     .action(async (opts) => {
       try {
+        if (!opts.global) {
+          const userId = getUserId();
+          try {
+            const subscriptions = await getUserSubscriptions(userId, { activeOnly: !opts.all });
+            if (opts.json) {
+              console.log(JSON.stringify(subscriptions.map((row) => ({
+                asin: row.product?.asin || null,
+                title: row.product?.title || null,
+                url: row.product?.url || null,
+                domain: row.product?.domain || null,
+                active: row.is_active,
+                tier: row.product?.tier || 'daily',
+                tierPref: row.tier_pref || null,
+                latestPrice: row.latestPrice ? parseFloat(row.latestPrice.price) : null,
+                currency: row.latestPrice?.currency || null,
+                lastScraped: row.latestPrice?.scraped_at || null,
+              }))));
+              return;
+            }
+
+            if (subscriptions.length === 0) {
+              console.log('No subscriptions found. Use `amaprice subscribe <url-or-asin>` to start tracking.');
+              return;
+            }
+
+            console.log(`Subscriptions for ${userId}:`);
+            for (const row of subscriptions) {
+              const price = row.latestPrice
+                ? formatPrice(parseFloat(row.latestPrice.price), row.latestPrice.currency)
+                : 'N/A';
+              const tier = row.product?.tier || 'daily';
+              const status = row.is_active ? tier : 'paused';
+              console.log(`  ${row.product?.asin || 'unknown'}  ${price}  [${status}]  ${row.product?.title || ''}`);
+            }
+            return;
+          } catch (err) {
+            const msg = String(err.message || '');
+            if (!/hybrid orchestration migration/i.test(msg)) {
+              throw err;
+            }
+            // Hybrid schema unavailable; fallback to legacy global listing.
+          }
+        }
+
         const products = await listProducts();
 
         if (opts.json) {

package/src/commands/subscribe.js

@@ -0,0 +1,126 @@
+const { normalizeAmazonInput } = require('../url');
+const { resolveCliInput } = require('../input');
+const { runCollectionPipeline } = require('../extractors/pipeline');
+const {
+  getProductByAsin,
+  upsertProduct,
+  insertPrice,
+  upsertProductLatestPrice,
+  updateProductById,
+  upsertUserSubscription,
+} = require('../db');
+const { getUserId } = require('../user-context');
+const { normalizeTier, computeNextScrapeAt } = require('../tiering');
+
+module.exports = function (program) {
+  program
+    .command('subscribe [input...]')
+    .description('Subscribe the current user to a tracked product (shared product catalog)')
+    .option('--tier <tier>', 'Preferred refresh tier for this subscription: hourly|daily|weekly')
+    .option('--json', 'Output as JSON')
+    .action(async (inputParts, opts) => {
+      const input = await resolveCliInput(inputParts);
+      const normalized = await normalizeAmazonInput(input);
+      if (!normalized) {
+        console.error('Error: Input must be an Amazon product URL or a valid ASIN.');
+        process.exit(1);
+      }
+
+      const selectedTier = opts.tier ? normalizeTier(opts.tier) : null;
+      if (opts.tier && !selectedTier) {
+        console.error('Error: Tier must be one of: hourly, daily, weekly.');
+        process.exit(1);
+      }
+
+      const userId = getUserId();
+
+      try {
+        let product = await getProductByAsin(normalized.asin);
+        let initial = null;
+
+        if (!product) {
+          initial = await runCollectionPipeline({
+            url: normalized.url,
+            domain: normalized.domain,
+            allowVision: true,
+            allowRailwayDomFallback: true,
+          });
+
+          product = await upsertProduct({
+            asin: normalized.asin,
+            title: initial.pageTitle || `ASIN ${normalized.asin}`,
+            url: normalized.url,
+            domain: normalized.domain,
+            tier: selectedTier || 'daily',
+            nextScrapeAt: computeNextScrapeAt(selectedTier || 'daily'),
+          });
+
+          if (initial.price) {
+            const priceRecord = await insertPrice({
+              productId: product.id,
+              price: initial.price.numeric,
+              currency: initial.price.currency,
+            });
+
+            await upsertProductLatestPrice({
+              productId: product.id,
+              price: initial.price.numeric,
+              currency: initial.price.currency,
+              scrapedAt: priceRecord.scraped_at,
+              source: initial.method,
+              confidence: initial.confidence,
+            }).catch(() => {});
+
+            await updateProductById(product.id, {
+              last_price: initial.price.numeric,
+              last_scraped_at: priceRecord.scraped_at,
+              consecutive_failures: 0,
+              last_error: null,
+              next_scrape_at: computeNextScrapeAt(selectedTier || product.tier || 'daily'),
+              last_price_change_at: priceRecord.scraped_at,
+            }).catch(() => {});
+          }
+        }
+
+        if (selectedTier && normalizeTier(product.tier) !== selectedTier) {
+          await updateProductById(product.id, {
+            tier: selectedTier,
+            next_scrape_at: computeNextScrapeAt(selectedTier),
+          }).catch(() => {});
+        }
+
+        const subscription = await upsertUserSubscription({
+          userId,
+          productId: product.id,
+          tierPref: selectedTier,
+          isActive: true,
+        });
+
+        if (opts.json) {
+          console.log(JSON.stringify({
+            userId,
+            subscriptionId: subscription.id,
+            active: subscription.is_active,
+            tierPref: subscription.tier_pref,
+            product: {
+              id: product.id,
+              asin: product.asin,
+              domain: product.domain,
+              url: product.url,
+              title: product.title,
+            },
+            initialPrice: initial?.price?.numeric || null,
+            initialCurrency: initial?.price?.currency || null,
+          }));
+          return;
+        }
+
+        console.log(`Subscribed: ${product.asin} (${product.title})`);
+        console.log(`User:       ${userId}`);
+        console.log(`Tier pref:  ${subscription.tier_pref || 'default'}`);
+      } catch (err) {
+        console.error(`Error: ${err.message}`);
+        process.exit(1);
+      }
+    });
+};

package/src/commands/subscriptions.js

@@ -0,0 +1,61 @@
+const { getUserSubscriptions } = require('../db');
+const { getUserId } = require('../user-context');
+const { formatPrice } = require('../format');
+
+module.exports = function (program) {
+  program
+    .command('subscriptions')
+    .description('List current user subscriptions and latest known prices')
+    .option('--all', 'Include inactive subscriptions')
+    .option('--json', 'Output as JSON')
+    .action(async (opts) => {
+      const userId = getUserId();
+
+      try {
+        const rows = await getUserSubscriptions(userId, {
+          activeOnly: !opts.all,
+        });
+
+        if (opts.json) {
+          console.log(JSON.stringify(rows.map((row) => ({
+            id: row.id,
+            userId: row.user_id,
+            active: row.is_active,
+            tierPref: row.tier_pref,
+            createdAt: row.created_at,
+            updatedAt: row.updated_at,
+            product: {
+              id: row.product?.id || null,
+              asin: row.product?.asin || null,
+              title: row.product?.title || null,
+              url: row.product?.url || null,
+              domain: row.product?.domain || null,
+              tier: row.product?.tier || null,
+            },
+            latestPrice: row.latestPrice ? Number(row.latestPrice.price) : null,
+            latestCurrency: row.latestPrice?.currency || null,
+            latestAt: row.latestPrice?.scraped_at || null,
+          }))));
+          return;
+        }
+
+        if (rows.length === 0) {
+          console.log('No subscriptions found. Use `amaprice subscribe <url-or-asin>` to add one.');
+          return;
+        }
+
+        console.log(`Subscriptions for ${userId}:`);
+        for (const row of rows) {
+          const price = row.latestPrice
+            ? formatPrice(Number(row.latestPrice.price), row.latestPrice.currency)
+            : 'N/A';
+          const subStatus = row.is_active ? 'active' : 'inactive';
+          const tier = row.product?.tier || 'daily';
+          console.log(`  ${row.product?.asin || 'unknown'}  ${price}  [${subStatus}] [tier=${tier}]  ${row.product?.title || ''}`);
+        }
+      } catch (err) {
+        console.error(`Error: ${err.message}`);
+        process.exit(1);
+      }
+    });
+};

package/src/commands/sync.js

@@ -5,12 +5,17 @@ module.exports = function (program) {
     .command('sync')
     .description('Run background sync for due products (for cron/worker usage)')
     .option('--limit <n>', 'Max products to process in one run', '20')
+    .option('--orchestrator', 'Use orchestrator queue flow')
+    .option('--legacy', 'Force legacy due-products flow')
     .option('--json', 'Output as JSON')
     .action(async (opts) => {
       const limit = Math.max(1, parseInt(opts.limit, 10) || 20);
+      const useOrchestrator = opts.legacy
+        ? false
+        : (opts.orchestrator ? true : undefined);
 
       try {
-        const report = await runDueSync({ limit });
+        const report = await runDueSync({ limit, useOrchestrator });
         if (opts.json) {
           console.log(JSON.stringify(report));
         } else {