amaprice 1.0.16 → 1.0.17

package/README.md

@@ -187,7 +187,9 @@ Run this SQL in Supabase SQL Editor:
 
 `supabase/migrations/20260223_enforce_collector_first_claiming.sql`
 
-These migrations add tier fields, indexes, telemetry, worker health rollups, `price_history.currency`, collector orchestration tables/functions, and strict collector-first claim policy.
+`supabase/migrations/20260225_add_web_track_requests.sql`
+
+These migrations add tier fields, indexes, telemetry, worker health rollups, `price_history.currency`, collector orchestration tables/functions, strict collector-first claim policy, and website intake request tracking.
 
 Note: these files are additive migrations and expect existing `products` + `price_history` tables.
 
@@ -260,6 +262,18 @@ Environment variables used by the npm package:
 
 For production background workers, prefer the Supabase **service role key**.
 
+Website API runtime (Next.js server routes):
+
+| Variable | Default | Used by | Notes |
+|---|---|---|---|
+| `SUPABASE_URL` | none | website API routes | Server-side Supabase project URL |
+| `SUPABASE_SERVICE_ROLE_KEY` | none | website API routes | Required for secure writes (`/api/v1/track-requests`) |
+| `WEB_TRACK_RATE_LIMIT_WINDOW_SECONDS` | `300` | website API routes | Sliding rate-limit window |
+| `WEB_TRACK_RATE_LIMIT_MAX_REQUESTS_PER_IP` | `8` | website API routes | Rate-limit threshold per IP hash |
+| `WEB_TRACK_RATE_LIMIT_MAX_REQUESTS_PER_VISITOR` | `12` | website API routes | Rate-limit threshold per visitor cookie |
+| `WEB_TRACK_IP_HASH_PEPPER` | `amaprice-default-pepper` | website API routes | Pepper for deterministic IP hashing |
+| `TURNSTILE_SECRET_KEY` | none | website API routes | Optional CAPTCHA verification for intake endpoint |
+
 ## Railway Worker Deployment
 
 This repo includes:
@@ -292,19 +306,28 @@ npm run worker:once
 
 ## Vercel Website Deployment (`amaprice.sh`)
 
-Lean marketing site is a Next.js app in `website/`.
+The website is a server-capable Next.js app in `website/` with API routes.
 
 Steps:
 1. Import the repo in Vercel.
 2. Leave the project at repo root (deployment is controlled by root `vercel.json`).
 3. Set website env vars:
-   - `NEXT_PUBLIC_SUPABASE_URL`
-   - `NEXT_PUBLIC_SUPABASE_ANON_KEY`
+   - `SUPABASE_URL`
+   - `SUPABASE_SERVICE_ROLE_KEY`
    - `NEXT_PUBLIC_GOOGLE_SITE_VERIFICATION` (for Google Search Console verification meta tag)
 4. Deploy.
 5. Add domain `amaprice.sh` in Vercel Domains and assign to this project.
 6. Set `www.amaprice.sh` redirect to `amaprice.sh`.
 
+### Website API Endpoints
+
+- `POST /api/v1/track-requests` - submit Amazon URL/ASIN intake request
+- `GET /api/v1/track-requests/:id` - poll request state (`queued`, `collecting`, `live`, ...)
+- `GET /api/v1/products` - live tracked products for website explorer
+- `GET /api/v1/products/:productId/history` - historical points for one product
+- `GET /api/v1/prices/recent` - recent global price points
+- `GET /api/v1/worker-health` - worker health rollup row
+
 Local website development:
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amaprice",
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "CLI tool to scrape and track Amazon product prices",
   "main": "src/scraper.js",
   "type": "commonjs",
@@ -39,5 +39,10 @@
     "playwright": "^1.58.2",
     "commander": "^13.1.0",
     "@supabase/supabase-js": "^2.49.4"
+  },
+  "devDependencies": {
+    "next": "16.1.6",
+    "react": "19.2.4",
+    "react-dom": "19.2.4"
   }
 }

package/src/db.js

@@ -54,6 +54,20 @@ function isMissingHybridSchema(error) {
   );
 }
 
+function isMissingWebTrackSchema(error) {
+  const code = String(error?.code || '');
+  const message = String(error?.message || '');
+  return (
+    code === '42P01'
+    || code === 'PGRST205'
+    || code === '42703'
+    || code === 'PGRST204'
+    || /relation .* does not exist/i.test(message)
+    || /column .* does not exist/i.test(message)
+    || /web_track_requests/i.test(message)
+  );
+}
+
 /**
  * Upsert a product by ASIN. Returns the product row.
  */
@@ -535,6 +549,51 @@ async function heartbeatCollector({
   return data;
 }
 
+async function listPendingWebTrackRequests(limit = 20) {
+  const supabase = getClient();
+  const safeLimit = Math.max(1, Number(limit) || 20);
+
+  const { data, error } = await supabase
+    .from('web_track_requests')
+    .select('id, raw_input, status, status_reason, request_meta, queued_at, created_at')
+    .eq('status', 'queued')
+    .is('product_id', null)
+    .order('created_at', { ascending: true })
+    .limit(safeLimit);
+
+  if (error) {
+    if (isMissingWebTrackSchema(error)) {
+      return [];
+    }
+    throw new Error(`Supabase web_track_requests error: ${error.message}`);
+  }
+
+  return data || [];
+}
+
+async function updateWebTrackRequestById(requestId, patch) {
+  const supabase = getClient();
+  const payload = cleanPayload({
+    ...patch,
+    updated_at: new Date().toISOString(),
+  });
+
+  const { data, error } = await supabase
+    .from('web_track_requests')
+    .update(payload)
+    .eq('id', requestId)
+    .select()
+    .single();
+
+  if (error) {
+    if (isMissingWebTrackSchema(error)) {
+      return null;
+    }
+    throw new Error(`Supabase web_track_requests error: ${error.message}`);
+  }
+  return data;
+}
+
 async function enqueueDueCollectionJobs(limit = 20) {
   const supabase = getClient();
   const safeLimit = Math.max(1, Number(limit) || 20);
@@ -753,6 +812,8 @@ module.exports = {
   upsertCollector,
   getCollectorById,
   heartbeatCollector,
+  listPendingWebTrackRequests,
+  updateWebTrackRequestById,
   enqueueDueCollectionJobs,
   claimCollectionJobs,
   completeCollectionJob,

package/src/orchestrator/runner.js

@@ -1,5 +1,6 @@
 const { runCollectionPipeline } = require('../extractors/pipeline');
 const {
+  getProductByAsin,
   enqueueDueCollectionJobs,
   claimCollectionJobs,
   completeCollectionJob,
@@ -7,10 +8,14 @@ const {
   insertCollectionAttempt,
   insertPrice,
   insertScrapeAttempt,
+  listPendingWebTrackRequests,
   upsertProductLatestPrice,
+  upsertProduct,
+  updateWebTrackRequestById,
   updateProductById,
   getRecentPrices,
 } = require('../db');
+const { normalizeAmazonInput, isAmazonUrl, extractDomain } = require('../url');
 const {
   normalizeTier,
   computeNextScrapeAt,
@@ -78,6 +83,119 @@ function nextJobStateAfterFailure(job) {
   return attempts >= maxAttempts ? 'dead' : 'queued';
 }
 
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function cleanJsonObject(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return {};
+  }
+  return { ...value };
+}
+
+async function resolvePendingWebTrackRequests({ limit = 20 } = {}) {
+  const safeLimit = Math.max(1, Number(limit) || 20);
+  const pending = await listPendingWebTrackRequests(safeLimit);
+  if (!pending || pending.length === 0) {
+    return { processed: 0, resolved: 0, rejected: 0 };
+  }
+
+  let resolved = 0;
+  let rejected = 0;
+
+  for (const request of pending) {
+    const rawInput = String(request.raw_input || '').trim();
+    if (!rawInput) {
+      rejected += 1;
+      await updateWebTrackRequestById(request.id, {
+        status: 'rejected',
+        status_reason: 'invalid_amazon_input',
+        last_error: 'Input is empty and could not be resolved.',
+        completed_at: nowIso(),
+        request_meta: {
+          ...cleanJsonObject(request.request_meta),
+          resolution_stage: 'collector',
+          resolution_error: 'empty_input',
+          resolution_finished_at: nowIso(),
+        },
+      }).catch(() => {});
+      continue;
+    }
+
+    const defaultDomain = isAmazonUrl(rawInput) ? extractDomain(rawInput) : 'amazon.de';
+    let normalized = null;
+    try {
+      normalized = await normalizeAmazonInput(rawInput, defaultDomain);
+    } catch {
+      normalized = null;
+    }
+
+    if (!normalized) {
+      rejected += 1;
+      await updateWebTrackRequestById(request.id, {
+        status: 'rejected',
+        status_reason: 'invalid_amazon_input',
+        last_error: 'Could not resolve Amazon URL to a product ASIN.',
+        completed_at: nowIso(),
+        request_meta: {
+          ...cleanJsonObject(request.request_meta),
+          resolution_stage: 'collector',
+          resolution_error: 'asin_not_found',
+          resolution_finished_at: nowIso(),
+        },
+      }).catch(() => {});
+      continue;
+    }
+
+    const existingProduct = await getProductByAsin(normalized.asin);
+    const now = nowIso();
+
+    const product = existingProduct
+      ? await updateProductById(existingProduct.id, {
+        is_active: true,
+        next_scrape_at: now,
+        domain: normalized.domain,
+        url: normalized.url,
+      })
+      : await upsertProduct({
+        asin: normalized.asin,
+        title: `ASIN ${normalized.asin}`,
+        url: normalized.url,
+        domain: normalized.domain,
+        tier: 'daily',
+        tierMode: 'auto',
+        isActive: true,
+        nextScrapeAt: now,
+      });
+
+    resolved += 1;
+    await updateWebTrackRequestById(request.id, {
+      asin: normalized.asin,
+      domain: normalized.domain,
+      normalized_url: normalized.url,
+      product_id: product.id,
+      status: 'queued',
+      status_reason: null,
+      last_error: null,
+      queued_at: request.queued_at || now,
+      completed_at: null,
+      request_meta: {
+        ...cleanJsonObject(request.request_meta),
+        resolution_stage: 'collector',
+        resolution_source: 'normalize_amazon_input',
+        resolution_finished_at: now,
+      },
+    }).catch(() => {});
+  }
+
+  return {
+    processed: pending.length,
+    resolved,
+    rejected,
+  };
+}
+
 async function processClaimedJob(job, {
   collectorId = null,
   executor = 'railway',
@@ -267,6 +385,9 @@ async function runOrchestratedSync({
 } = {}) {
   const safeLimit = Math.max(1, Number(limit) || 20);
 
+  await resolvePendingWebTrackRequests({
+    limit: Math.max(safeLimit * 2, safeLimit),
+  }).catch(() => {});
   await requeueExpiredCollectionJobs(200).catch(() => {});
   await enqueueDueCollectionJobs(Math.max(safeLimit * 2, safeLimit)).catch(() => {});
 
@@ -319,4 +440,5 @@ module.exports.__test = {
   classifyFailure,
   buildNoPriceErrorMessage,
   nextJobStateAfterFailure,
+  cleanJsonObject,
 };

package/src/url.js

@@ -5,7 +5,37 @@ const AMAZON_DOMAINS = [
 ];
 const AMAZON_SHORT_DOMAINS = ['amzn.eu', 'amzn.to', 'a.co'];
 
-const ASIN_REGEX = /(?:\/(?:dp|gp\/product|ASIN)\/)([A-Z0-9]{10})/i;
+const ASIN_PATH_REGEX = /(?:\/(?:dp|gp\/product|gp\/aw\/d|ASIN)\/)([A-Z0-9]{10})(?=[/?]|$)/i;
+const DIRECT_ASIN_REGEX = /^[A-Z0-9]{10}$/i;
+const QUERY_ASIN_KEYS = new Set(['asin', 'pd_rd_i']);
+const QUERY_NESTED_URL_KEYS = new Set(['url', 'u', 'redirecturl', 'path']);
+const HTML_ASIN_PATTERNS = [
+  /<link[^>]+rel=["']canonical["'][^>]+href=["'][^"']*(?:\/dp\/|\/gp\/product\/|\/gp\/aw\/d\/)([A-Z0-9]{10})(?=[/?"'&]|$)/i,
+  /<link[^>]+href=["'][^"']*(?:\/dp\/|\/gp\/product\/|\/gp\/aw\/d\/)([A-Z0-9]{10})(?=[/?"'&]|$)[^>]*rel=["']canonical["']/i,
+  /"(?:currentAsin|parentAsin|landingAsin)"\s*:\s*"([A-Z0-9]{10})"/i,
+  /"(?:canonicalUrl|productUrl|dpUrl|redirectUrl)"\s*:\s*"[^"]*(?:\/dp\/|\/gp\/product\/|\/gp\/aw\/d\/)([A-Z0-9]{10})(?=[/?"&]|$)/i,
+];
+const FETCH_TIMEOUT_MS = 9000;
+const BROWSERISH_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36';
+
+const DOMAIN_ACCEPT_LANGUAGE = {
+  'amazon.de': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.fr': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.it': 'it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.es': 'es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.nl': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.co.jp': 'ja-JP,ja;q=0.9,en-US;q=0.7,en;q=0.6',
+  'amazon.com.br': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
+  'amazon.co.uk': 'en-GB,en;q=0.9',
+  'amazon.com': 'en-US,en;q=0.9',
+  'amazon.ca': 'en-CA,en;q=0.9,fr-CA;q=0.6',
+  'amazon.com.au': 'en-AU,en;q=0.9',
+  'amazon.in': 'en-IN,en;q=0.9',
+};
+
+function safeTrim(value) {
+  return String(value || '').trim();
+}
 
 function isAmazonUrl(url) {
   try {
@@ -19,22 +49,97 @@ function isAmazonUrl(url) {
 function isAmazonShortUrl(url) {
   try {
     const parsed = new URL(url);
-    const hostname = parsed.hostname.replace(/^www\./, '');
+    const hostname = parsed.hostname.replace(/^www\./i, '').toLowerCase();
     return AMAZON_SHORT_DOMAINS.includes(hostname);
   } catch {
     return false;
   }
 }
 
-function extractAsin(urlOrAsin) {
-  // If it's already a bare ASIN (10 alphanumeric chars)
-  if (/^[A-Z0-9]{10}$/i.test(urlOrAsin)) {
-    return urlOrAsin.toUpperCase();
+function extractAsinFromPathLikeValue(value) {
+  const trimmed = safeTrim(value);
+  if (!trimmed) {
+    return null;
   }
-  const match = urlOrAsin.match(ASIN_REGEX);
+
+  if (DIRECT_ASIN_REGEX.test(trimmed)) {
+    return trimmed.toUpperCase();
+  }
+
+  const match = trimmed.match(ASIN_PATH_REGEX);
   return match ? match[1].toUpperCase() : null;
 }
 
+function extractAsinFromQueryParams(parsed) {
+  for (const [rawKey, rawValue] of parsed.searchParams.entries()) {
+    const key = safeTrim(rawKey).toLowerCase();
+    const value = safeTrim(rawValue);
+    if (!key || !value) {
+      continue;
+    }
+
+    if (QUERY_ASIN_KEYS.has(key) && DIRECT_ASIN_REGEX.test(value)) {
+      return value.toUpperCase();
+    }
+
+    if (!QUERY_NESTED_URL_KEYS.has(key)) {
+      continue;
+    }
+
+    const nestedDirect = extractAsinFromPathLikeValue(value);
+    if (nestedDirect) {
+      return nestedDirect;
+    }
+
+    try {
+      const decoded = decodeURIComponent(value);
+      const nestedDecoded = extractAsinFromPathLikeValue(decoded);
+      if (nestedDecoded) {
+        return nestedDecoded;
+      }
+    } catch {
+      // Ignore decode failures for non-encoded payloads.
+    }
+  }
+
+  return null;
+}
+
+function extractAsinFromHtml(html) {
+  const body = safeTrim(html);
+  if (!body) {
+    return null;
+  }
+
+  for (const pattern of HTML_ASIN_PATTERNS) {
+    const match = body.match(pattern);
+    if (match?.[1]) {
+      return match[1].toUpperCase();
+    }
+  }
+
+  return null;
+}
+
+function extractAsin(urlOrAsin) {
+  const value = safeTrim(urlOrAsin);
+  if (!value) {
+    return null;
+  }
+
+  const direct = extractAsinFromPathLikeValue(value);
+  if (direct) {
+    return direct;
+  }
+
+  try {
+    const parsed = new URL(value);
+    return extractAsinFromQueryParams(parsed);
+  } catch {
+    return null;
+  }
+}
+
 function extractDomain(url) {
   try {
     const parsed = new URL(url);
@@ -48,39 +153,175 @@ function canonicalUrl(asin, domain = 'amazon.de') {
   return `https://www.${domain}/dp/${asin}`;
 }
 
+function isResolvableAmazonUrl(url) {
+  return isAmazonShortUrl(url) || isAmazonUrl(url);
+}
+
+function defaultDomainForUrl(url) {
+  return isAmazonUrl(url) ? extractDomain(url) : 'amazon.de';
+}
+
+function buildResolveHeaders(url) {
+  const domain = defaultDomainForUrl(url);
+  return {
+    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'accept-language': DOMAIN_ACCEPT_LANGUAGE[domain] || 'en-US,en;q=0.9',
+    'cache-control': 'no-cache',
+    pragma: 'no-cache',
+    'user-agent': BROWSERISH_USER_AGENT,
+  };
+}
+
+async function fetchWithTimeout(url, options = {}) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => {
+    controller.abort();
+  }, FETCH_TIMEOUT_MS);
+
+  try {
+    return await fetch(url, {
+      ...options,
+      signal: controller.signal,
+    });
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+async function extractAsinFromResponse(response) {
+  const contentType = response.headers?.get
+    ? String(response.headers.get('content-type') || '').toLowerCase()
+    : '';
+  if (contentType && !contentType.includes('text/html')) {
+    return null;
+  }
+
+  if (typeof response.text !== 'function') {
+    return null;
+  }
+
+  try {
+    const html = await response.text();
+    return extractAsinFromHtml(String(html || '').slice(0, 512_000));
+  } catch {
+    return null;
+  }
+}
+
+function resolveAsinFromAmazonUrl(url, fallbackDomain = 'amazon.de') {
+  const asin = extractAsin(url);
+  if (!asin || !isAmazonUrl(url)) {
+    return null;
+  }
+
+  const domain = extractDomain(url) || fallbackDomain;
+  return canonicalUrl(asin, domain);
+}
+
+async function resolveByFollowingRedirects(url) {
+  const originDomain = defaultDomainForUrl(url);
+  const response = await fetchWithTimeout(url, {
+    method: 'GET',
+    redirect: 'follow',
+    headers: buildResolveHeaders(url),
+  });
+  if (!response) {
+    return null;
+  }
+
+  const responseUrl = safeTrim(response.url);
+  const fromUrl = resolveAsinFromAmazonUrl(responseUrl, originDomain);
+  if (fromUrl) {
+    try {
+      await response.body?.cancel?.();
+    } catch {
+      // Ignore body cancellation failures.
+    }
+    return fromUrl;
+  }
+
+  const htmlAsin = await extractAsinFromResponse(response);
+  if (!htmlAsin) {
+    return null;
+  }
+
+  const domain = isAmazonUrl(responseUrl)
+    ? extractDomain(responseUrl)
+    : originDomain;
+  return canonicalUrl(htmlAsin, domain);
+}
+
 async function resolveAmazonShortUrl(url, maxRedirects = 8) {
-  if (!isAmazonShortUrl(url)) return url;
+  if (!isResolvableAmazonUrl(url)) {
+    return url;
+  }
+
+  const followed = await resolveByFollowingRedirects(url);
+  if (followed) {
+    return followed;
+  }
 
   let current = url;
+  const originDomain = defaultDomainForUrl(url);
+  const visited = new Set();
   for (let i = 0; i < maxRedirects; i += 1) {
-    let response;
-    try {
-      response = await fetch(current, {
-        method: 'GET',
-        redirect: 'manual',
-      });
-    } catch {
+    if (visited.has(current)) {
+      return current;
+    }
+    visited.add(current);
+
+    const response = await fetchWithTimeout(current, {
+      method: 'GET',
+      redirect: 'manual',
+      headers: buildResolveHeaders(current),
+    });
+    if (!response) {
       return current;
     }
 
+    const fromResponseUrl = resolveAsinFromAmazonUrl(
+      safeTrim(response.url),
+      originDomain,
+    );
+    if (fromResponseUrl) {
+      try {
+        await response.body?.cancel?.();
+      } catch {
+        // Ignore body cancellation failures.
+      }
+      return fromResponseUrl;
+    }
+
     const location = response.headers?.get ? response.headers.get('location') : null;
+    if (!location || response.status < 300 || response.status > 399) {
+      const htmlAsin = await extractAsinFromResponse(response);
+      if (htmlAsin) {
+        const domain = isAmazonUrl(current) ? extractDomain(current) : originDomain;
+        return canonicalUrl(htmlAsin, domain);
+      }
+      return current;
+    }
+
     try {
       await response.body?.cancel?.();
     } catch {
       // Ignore body cancellation failures.
     }
 
-    if (!location || response.status < 300 || response.status > 399) {
-      return current;
-    }
-
     try {
       current = new URL(location, current).toString();
     } catch {
       return current;
     }
 
-    if (isAmazonUrl(current) && extractAsin(current)) {
+    const fromLocation = resolveAsinFromAmazonUrl(current, originDomain);
+    if (fromLocation) {
+      return fromLocation;
+    }
+
+    if (!isResolvableAmazonUrl(current)) {
       return current;
     }
   }
@@ -89,7 +330,7 @@ async function resolveAmazonShortUrl(url, maxRedirects = 8) {
 }
 
 async function normalizeAmazonInput(input, defaultDomain = 'amazon.de') {
-  const raw = String(input || '').trim();
+  const raw = safeTrim(input);
   if (!raw) return null;
 
   const asin = extractAsin(raw);