idle-node 0.1.0 → 0.2.0

package/bin/idle-node.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
-import { startNode } from '../src/node.js';
+import { startNode, ALL_TYPES } from '../src/node.js';
 import chalk from 'chalk';
 
 const program = new Command();
@@ -8,24 +8,18 @@ const program = new Command();
 program
   .name('idle-node')
   .description('IDLE Protocol node — earn from idle compute')
-  .version('0.1.0');
+  .version('0.2.0');
 
 program
   .command('start')
   .description('Start the IDLE node and begin processing jobs')
   .requiredOption('--wallet <address>', 'Solana wallet address for payouts')
-  .option('--types <types>', 'Comma-separated job types to accept (default: fetch,validate)', 'fetch,validate')
+  .option('--types <types>', 'Comma-separated job types to accept (default: all)', ALL_TYPES.join(','))
   .option('--poll-interval <ms>', 'Job poll interval in milliseconds', '5000')
   .option('--max-concurrent <n>', 'Max concurrent jobs', '3')
   .option('--supabase-url <url>', 'Supabase project URL', process.env.IDLE_SUPABASE_URL || 'https://vnhzyynewdtfpiynaaqd.supabase.co')
   .option('--supabase-key <key>', 'Supabase anon key', process.env.IDLE_SUPABASE_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZuaHp5eW5ld2R0ZnBpeW5hYXFkIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzY5NzgyMDksImV4cCI6MjA5MjU1NDIwOX0.6Sw8p9wnfx67NjhzJGmPWialMG4517Eubj9YyfHPaJc')
   .action(async (opts) => {
-    console.log(chalk.green('IDLE Node v0.1.0'));
-    console.log(chalk.gray(`Wallet: ${opts.wallet}`));
-    console.log(chalk.gray(`Types: ${opts.types}`));
-    console.log(chalk.gray(`Poll interval: ${opts.pollInterval}ms`));
-    console.log('');
-
     try {
       await startNode({
         wallet: opts.wallet,
@@ -52,4 +46,22 @@ program
     await checkStatus(opts);
   });
 
+program
+  .command('types')
+  .description('List all supported task types')
+  .action(() => {
+    console.log(chalk.green.bold('\n  IDLE Protocol — Supported Task Types\n'));
+    const tiers = {
+      'Tier 1 — Basic Monitoring': ['http_fetch', 'api_health', 'dns_lookup', 'ssl_check', 'response_time', 'json_validate'],
+      'Tier 2 — Web Intelligence': ['html_scrape', 'element_extract', 'link_extract', 'price_extract', 'availability_check', 'content_verify', 'change_detect'],
+    };
+    for (const [tier, types] of Object.entries(tiers)) {
+      console.log(chalk.cyan(`  ${tier}`));
+      for (const t of types) {
+        console.log(chalk.white(`    - ${t}`));
+      }
+      console.log('');
+    }
+  });
+
 program.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "idle-node",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "IDLE Protocol node — turn idle compute into revenue",
   "main": "./src/node.js",
   "bin": {
@@ -29,7 +29,12 @@
   },
   "dependencies": {
     "@supabase/supabase-js": "^2.45.0",
+    "axios": "^1.16.0",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.2.0",
     "commander": "^12.1.0",
-    "chalk": "^5.3.0"
+    "puppeteer": "^24.43.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   }
 }

package/src/node.js

@@ -1,39 +1,42 @@
 import { createClient } from '@supabase/supabase-js';
 import chalk from 'chalk';
+import {
+  executeHttpFetch,
+  executeHtmlScrape,
+  executeElementExtract,
+  executeLinkExtract,
+  executePriceExtract,
+  executeAvailabilityCheck,
+  executeContentVerify,
+  executeDnsLookup,
+  executeSslCheck,
+  executeResponseTime,
+  executeApiHealth,
+  executeJsonValidate,
+  executeChangeDetect,
+} from './tasks/fast.js';
 
 const EXECUTORS = {
-  fetch: async (payload) => {
-    const { url, method = 'GET' } = payload;
-    const start = Date.now();
-    const res = await fetch(url, { method });
-    const body = await res.text();
-    return {
-      status: res.status,
-      headers: Object.fromEntries(res.headers.entries()),
-      body: body.slice(0, 10000),
-      latency_ms: Date.now() - start,
-    };
-  },
-
-  validate: async (payload) => {
-    const { data, schema } = payload;
-    // Basic JSON schema validation
-    const errors = [];
-    if (schema?.required) {
-      for (const field of schema.required) {
-        if (!(field in (data || {}))) errors.push(`missing required field: ${field}`);
-      }
-    }
-    return { valid: errors.length === 0, errors };
-  },
-
-  query: async (payload) => {
-    // Natural language query placeholder — real implementation
-    // would connect to user's configured data source
-    return { error: 'query execution requires data source configuration' };
-  },
+  http_fetch: executeHttpFetch,
+  fetch: executeHttpFetch,
+  html_scrape: executeHtmlScrape,
+  element_extract: executeElementExtract,
+  link_extract: executeLinkExtract,
+  price_extract: executePriceExtract,
+  availability_check: executeAvailabilityCheck,
+  content_verify: executeContentVerify,
+  change_detect: executeChangeDetect,
+  dns_lookup: executeDnsLookup,
+  ssl_check: executeSslCheck,
+  response_time: executeResponseTime,
+  api_health: executeApiHealth,
+  json_validate: executeJsonValidate,
+  validate: executeJsonValidate,
+  query: async (payload) => ({ error: 'query execution requires data source configuration' }),
 };
 
+const ALL_TYPES = Object.keys(EXECUTORS).filter(k => k !== 'fetch' && k !== 'validate');
+
 export async function startNode(config) {
   if (!config.supabaseUrl || !config.supabaseKey) {
     throw new Error('Supabase URL and key required. Set IDLE_SUPABASE_URL and IDLE_SUPABASE_KEY env vars, or pass --supabase-url and --supabase-key.');
@@ -44,13 +47,22 @@ export async function startNode(config) {
   let totalProcessed = 0;
   let totalEarned = 0;
 
-  console.log(chalk.green('Node started. Polling for jobs...'));
+  console.log('');
+  console.log(chalk.green.bold('  IDLE Protocol Node v0.2.0'));
+  console.log(chalk.gray('  The earnings layer for idle compute'));
+  console.log('');
+  console.log(chalk.white(`  Wallet:  ${config.wallet}`));
+  console.log(chalk.white(`  Types:   ${config.types.length} task types`));
+  console.log(chalk.white(`  Poll:    ${config.pollInterval}ms`));
+  console.log(chalk.white(`  Workers: ${config.maxConcurrent} concurrent`));
+  console.log('');
+  console.log(chalk.cyan('  Waiting for tasks...'));
+  console.log('');
 
   const poll = async () => {
     if (running >= config.maxConcurrent) return;
 
     try {
-      // Claim a pending job atomically
       const { data: jobs, error } = await supabase
         .rpc('idle_claim_node_job', {
           p_node_wallet: config.wallet,
@@ -59,7 +71,9 @@ export async function startNode(config) {
         });
 
       if (error) {
-        console.error(chalk.red('Poll error:'), error.message);
+        if (!error.message.includes('does not exist')) {
+          console.error(chalk.red('  Poll error:'), error.message);
+        }
         return;
       }
 
@@ -71,34 +85,29 @@ export async function startNode(config) {
           running--;
           totalProcessed++;
           totalEarned += earned;
-          if (totalProcessed % 10 === 0) {
-            console.log(chalk.cyan(`[stats] ${totalProcessed} jobs | $${totalEarned.toFixed(4)} earned`));
-          }
+          console.log(chalk.green(`  Done | Earned: $${earned.toFixed(4)} | Total: $${totalEarned.toFixed(4)} | Jobs: ${totalProcessed}`));
         }).catch((err) => {
           running--;
-          console.error(chalk.red(`Job ${job.id} failed:`), err.message);
+          console.error(chalk.red(`  Job ${job.id.slice(0, 8)} failed:`), err.message);
         });
       }
     } catch (err) {
-      console.error(chalk.red('Poll error:'), err.message);
+      console.error(chalk.red('  Poll error:'), err.message);
     }
   };
 
-  // Poll loop
   const interval = setInterval(poll, config.pollInterval);
 
-  // Graceful shutdown
   const shutdown = () => {
-    console.log(chalk.yellow('\nShutting down...'));
+    console.log(chalk.yellow('\n  Shutting down...'));
     clearInterval(interval);
-    console.log(chalk.green(`Processed ${totalProcessed} jobs, earned $${totalEarned.toFixed(4)}`));
+    console.log(chalk.green(`  Processed ${totalProcessed} jobs, earned $${totalEarned.toFixed(4)}`));
     process.exit(0);
   };
 
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);
 
-  // Initial poll
   await poll();
 }
 
@@ -113,7 +122,7 @@ async function processJob(supabase, job, wallet) {
   }
 
   const start = Date.now();
-  console.log(chalk.gray(`[${job.type}] Processing ${job.id.slice(0, 8)}...`));
+  console.log(chalk.gray(`  [${job.type}] Processing ${job.id.slice(0, 8)}...`));
 
   try {
     const result = await executor(job.payload);
@@ -131,7 +140,7 @@ async function processJob(supabase, job, wallet) {
       .eq('id', job.id);
 
     const earned = job.node_payout_usd || 0;
-    console.log(chalk.green(`[${job.type}] ${job.id.slice(0, 8)} done (${duration}ms) +$${earned.toFixed(4)}`));
+    console.log(chalk.green(`  [${job.type}] ${job.id.slice(0, 8)} done (${duration}ms) +$${earned.toFixed(4)}`));
     return earned;
   } catch (err) {
     await supabase
@@ -146,3 +155,5 @@ async function processJob(supabase, job, wallet) {
     throw err;
   }
 }
+
+export { ALL_TYPES };

package/src/tasks/fast.js

@@ -0,0 +1,396 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import dns from 'dns/promises';
+import crypto from 'crypto';
+import https from 'https';
+
+const DEFAULT_HEADERS = {
+  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.5',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'DNT': '1',
+  'Connection': 'keep-alive',
+};
+
+export async function executeHttpFetch(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios({
+      url: payload.url,
+      method: (payload.method || 'GET'),
+      headers: { ...DEFAULT_HEADERS, ...(payload.headers || {}) },
+      timeout: 10000,
+      maxRedirects: 5,
+      validateStatus: () => true,
+    });
+    return {
+      status: res.status,
+      body: typeof res.data === 'string' ? res.data.slice(0, 10000) : JSON.stringify(res.data).slice(0, 10000),
+      headers: res.headers,
+      latency_ms: Date.now() - start,
+      success: true,
+    };
+  } catch (err) {
+    return { status: 0, body: err.message, latency_ms: Date.now() - start, success: false };
+  }
+}
+
+export async function executeHtmlScrape(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, {
+      headers: DEFAULT_HEADERS,
+      timeout: 15000,
+      maxRedirects: 5,
+    });
+    const $ = cheerio.load(res.data);
+    $('script, style, noscript, iframe').remove();
+
+    let result;
+    if (payload.selector) {
+      result = $(payload.selector).html() || '';
+    } else {
+      result = $.html();
+    }
+
+    return {
+      status: res.status,
+      html: result.slice(0, 50000),
+      text: $('body').text().replace(/\s+/g, ' ').trim().slice(0, 20000),
+      title: $('title').text(),
+      latency_ms: Date.now() - start,
+      success: true,
+    };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeElementExtract(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+    const extracted = {};
+
+    for (const [key, selector] of Object.entries(payload.selectors)) {
+      const elements = $(selector);
+      const attr = payload.attribute || 'text';
+
+      if (attr === 'text') {
+        extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+      } else if (attr === 'html') {
+        extracted[key] = elements.map((_, el) => $(el).html() || '').get();
+      } else {
+        extracted[key] = elements.map((_, el) => $(el).attr(attr) || '').get();
+      }
+
+      if (Array.isArray(extracted[key]) && extracted[key].length === 1) {
+        extracted[key] = extracted[key][0];
+      }
+    }
+
+    return { success: true, extracted, latency_ms: Date.now() - start };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeLinkExtract(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+    const baseUrl = new URL(payload.url);
+    const links = [];
+
+    $('a[href]').each((_, el) => {
+      const href = $(el).attr('href') || '';
+      const text = $(el).text().trim();
+      try {
+        const url = new URL(href, payload.url);
+        const type = url.hostname === baseUrl.hostname ? 'internal' : 'external';
+        if (!payload.filter || payload.filter === 'all' || payload.filter === type) {
+          links.push({ href: url.href, text, type });
+        }
+      } catch {}
+    });
+
+    return {
+      success: true,
+      links: links.slice(0, 500),
+      total: links.length,
+      latency_ms: Date.now() - start,
+    };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executePriceExtract(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+
+    const priceSelectors = [
+      '[itemprop="price"]',
+      '.price', '.product-price', '#price', '.offer-price',
+      '[class*="price"]', '[id*="price"]',
+      'span[data-price]', 'meta[property="product:price:amount"]',
+    ];
+
+    let price = null;
+    let currency = null;
+
+    for (const sel of priceSelectors) {
+      const el = $(sel).first();
+      if (el.length) {
+        price = el.attr('content') || el.attr('data-price') || el.text().trim();
+        if (price && /[\d]/.test(price)) break;
+      }
+    }
+
+    currency = $('meta[property="product:price:currency"]').attr('content') ||
+               $('[itemprop="priceCurrency"]').attr('content') || null;
+
+    $('script[type="application/ld+json"]').each((_, el) => {
+      try {
+        const data = JSON.parse($(el).html() || '{}');
+        if (data.offers?.price && !price) price = String(data.offers.price);
+        if (data.offers?.priceCurrency && !currency) currency = data.offers.priceCurrency;
+      } catch {}
+    });
+
+    return { success: true, price, currency, raw: price, latency_ms: Date.now() - start };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeAvailabilityCheck(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+    const html = res.data.toLowerCase();
+
+    let availability = null;
+    $('script[type="application/ld+json"]').each((_, el) => {
+      try {
+        const data = JSON.parse($(el).html() || '{}');
+        if (data.offers?.availability) availability = data.offers.availability;
+      } catch {}
+    });
+
+    const inStockSignals = ['in stock', 'add to cart', 'buy now', 'add to bag', 'in-stock'];
+    const outOfStockSignals = ['out of stock', 'sold out', 'unavailable', 'not available', 'backorder'];
+
+    let inStock = null;
+    if (availability) {
+      inStock = availability.toLowerCase().includes('instock');
+    } else {
+      const isInStock = inStockSignals.some(s => html.includes(s));
+      const isOutOfStock = outOfStockSignals.some(s => html.includes(s));
+      if (isInStock && !isOutOfStock) inStock = true;
+      else if (isOutOfStock) inStock = false;
+    }
+
+    return { success: true, in_stock: inStock, availability_raw: availability, latency_ms: Date.now() - start };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeContentVerify(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+    const bodyText = $('body').text();
+
+    let found = false;
+    let matchedText = null;
+
+    if (payload.text) {
+      found = bodyText.toLowerCase().includes(payload.text.toLowerCase());
+      if (found) matchedText = payload.text;
+    }
+
+    if (payload.pattern) {
+      const regex = new RegExp(payload.pattern, 'i');
+      const match = bodyText.match(regex);
+      if (match) {
+        found = true;
+        matchedText = match[0];
+      }
+    }
+
+    if (payload.selector) {
+      const el = $(payload.selector);
+      found = el.length > 0;
+      matchedText = el.first().text().trim();
+    }
+
+    return { success: true, found, matched_text: matchedText, status: res.status, latency_ms: Date.now() - start };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeDnsLookup(payload) {
+  const start = Date.now();
+  try {
+    const type = payload.type || 'A';
+    let records;
+
+    if (type === 'A') records = await dns.resolve4(payload.hostname);
+    else if (type === 'AAAA') records = await dns.resolve6(payload.hostname);
+    else if (type === 'MX') records = await dns.resolveMx(payload.hostname);
+    else if (type === 'TXT') records = await dns.resolveTxt(payload.hostname);
+    else if (type === 'NS') records = await dns.resolveNs(payload.hostname);
+    else records = await dns.resolve(payload.hostname);
+
+    return { success: true, hostname: payload.hostname, type, records, latency_ms: Date.now() - start };
+  } catch (err) {
+    return { success: false, hostname: payload.hostname, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeSslCheck(payload) {
+  const start = Date.now();
+  return new Promise(resolve => {
+    const req = https.request({ hostname: payload.hostname, port: 443, method: 'HEAD', rejectUnauthorized: false }, (res) => {
+      const cert = res.socket.getPeerCertificate();
+      const now = new Date();
+      const expiresAt = new Date(cert.valid_to);
+      const daysUntilExpiry = Math.floor((expiresAt.getTime() - now.getTime()) / (1000 * 60 * 60 * 24));
+
+      resolve({
+        success: true,
+        hostname: payload.hostname,
+        valid: res.socket.authorized,
+        issuer: cert.issuer?.CN || cert.issuer?.O,
+        subject: cert.subject?.CN,
+        expires_at: cert.valid_to,
+        days_until_expiry: daysUntilExpiry,
+        expired: daysUntilExpiry < 0,
+        latency_ms: Date.now() - start,
+      });
+      req.destroy();
+    });
+    req.on('error', (err) => {
+      resolve({ success: false, hostname: payload.hostname, error: err.message, latency_ms: Date.now() - start });
+    });
+    req.setTimeout(10000, () => {
+      req.destroy();
+      resolve({ success: false, hostname: payload.hostname, error: 'timeout', latency_ms: Date.now() - start });
+    });
+    req.end();
+  });
+}
+
+export async function executeResponseTime(payload) {
+  const iterations = Math.min(payload.iterations || 3, 5);
+  const times = [];
+  let lastStatus = 0;
+
+  for (let i = 0; i < iterations; i++) {
+    const start = Date.now();
+    try {
+      const res = await axios.get(payload.url, {
+        headers: DEFAULT_HEADERS,
+        timeout: 10000,
+        validateStatus: () => true,
+      });
+      lastStatus = res.status;
+      times.push(Date.now() - start);
+    } catch {
+      times.push(10000);
+    }
+    if (i < iterations - 1) await new Promise(r => setTimeout(r, 200));
+  }
+
+  const sorted = [...times].sort((a, b) => a - b);
+  return {
+    success: true,
+    url: payload.url,
+    status: lastStatus,
+    p50_ms: sorted[Math.floor(sorted.length / 2)],
+    p95_ms: sorted[Math.floor(sorted.length * 0.95)],
+    min_ms: Math.min(...times),
+    max_ms: Math.max(...times),
+    avg_ms: Math.round(times.reduce((a, b) => a + b, 0) / times.length),
+    iterations,
+  };
+}
+
+export async function executeApiHealth(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios({
+      url: payload.url,
+      method: (payload.method || 'GET'),
+      headers: DEFAULT_HEADERS,
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+
+    const body = typeof res.data === 'string' ? res.data : JSON.stringify(res.data);
+    const statusOk = !payload.expected_status || res.status === payload.expected_status;
+    const bodyOk = !payload.expected_body || body.includes(payload.expected_body);
+
+    return {
+      success: true,
+      healthy: statusOk && bodyOk,
+      status: res.status,
+      status_ok: statusOk,
+      body_ok: bodyOk,
+      latency_ms: Date.now() - start,
+      body_preview: body.slice(0, 500),
+    };
+  } catch (err) {
+    return { success: false, healthy: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}
+
+export async function executeJsonValidate(payload) {
+  const errors = [];
+  for (const [key, expectedType] of Object.entries(payload.schema)) {
+    const val = payload.data?.[key];
+    if (val === undefined || val === null) {
+      errors.push(`Missing field: ${key}`);
+    } else if (typeof val !== expectedType) {
+      errors.push(`${key}: expected ${expectedType}, got ${typeof val}`);
+    }
+  }
+  return { valid: errors.length === 0, errors, field_count: Object.keys(payload.schema).length };
+}
+
+export async function executeChangeDetect(payload) {
+  const start = Date.now();
+  try {
+    const res = await axios.get(payload.url, { headers: DEFAULT_HEADERS, timeout: 15000 });
+    const $ = cheerio.load(res.data);
+    $('script, style, noscript').remove();
+
+    const content = payload.selector
+      ? $(payload.selector).text().replace(/\s+/g, ' ').trim()
+      : $('body').text().replace(/\s+/g, ' ').trim();
+
+    const hash = crypto.createHash('sha256').update(content).digest('hex');
+    const changed = payload.previous_hash ? hash !== payload.previous_hash : null;
+
+    return {
+      success: true,
+      hash,
+      changed,
+      content_length: content.length,
+      content_preview: content.slice(0, 200),
+      latency_ms: Date.now() - start,
+    };
+  } catch (err) {
+    return { success: false, error: err.message, latency_ms: Date.now() - start };
+  }
+}