contextractor 0.1.0

package/cli.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+"use strict";
+
+const { spawn } = require("child_process");
+const { getBinaryPath } = require("./index");
+
+const binaryPath = getBinaryPath();
+const child = spawn(binaryPath, process.argv.slice(2), {
+  stdio: "inherit",
+});
+
+child.on("error", (err) => {
+  if (err.code === "ENOENT") {
+    console.error(
+      `contextractor binary not found at ${binaryPath}\n` +
+        "Try reinstalling: npm install contextractor"
+    );
+    process.exit(1);
+  }
+  throw err;
+});
+
+child.on("exit", (code) => {
+  process.exit(code ?? 1);
+});

package/index.js

@@ -0,0 +1,65 @@
+"use strict";
+
+const path = require("path");
+const os = require("os");
+const { execFileSync, spawn } = require("child_process");
+
+const PLATFORM_MAP = {
+  darwin: "darwin",
+  linux: "linux",
+  win32: "win",
+};
+
+const ARCH_MAP = {
+  x64: "x64",
+  arm64: "arm64",
+};
+
+function getBinaryName() {
+  const platform = PLATFORM_MAP[os.platform()];
+  let arch = ARCH_MAP[os.arch()];
+
+  if (!platform || !arch) {
+    throw new Error(
+      `Unsupported platform: ${os.platform()}-${os.arch()}\n` +
+        "Supported: darwin-arm64, linux-x64, linux-arm64, win-x64"
+    );
+  }
+
+  // macOS x64 uses arm64 binary via Rosetta
+  if (platform === "darwin" && arch === "x64") {
+    arch = "arm64";
+  }
+
+  const ext = os.platform() === "win32" ? ".exe" : "";
+  return `contextractor-${platform}-${arch}${ext}`;
+}
+
+function getBinaryPath() {
+  return path.join(__dirname, "bin", getBinaryName());
+}
+
+function extract(configPath, options = {}) {
+  return new Promise((resolve, reject) => {
+    const args = [configPath];
+    if (options.precision) args.push("--precision");
+    if (options.recall) args.push("--recall");
+    if (options.noLinks) args.push("--no-links");
+    if (options.noComments) args.push("--no-comments");
+    if (options.outputDir) args.push("--output-dir", options.outputDir);
+    if (options.format) args.push("--format", options.format);
+    if (options.verbose) args.push("--verbose");
+
+    const child = spawn(getBinaryPath(), args, {
+      stdio: options.stdio || "inherit",
+    });
+
+    child.on("error", reject);
+    child.on("exit", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`contextractor exited with code ${code}`));
+    });
+  });
+}
+
+module.exports = { getBinaryPath, getBinaryName, extract };

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "contextractor",
+  "version": "0.1.0",
+  "description": "Extract web content from URLs with configurable extraction options",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/contextractor/contextractor.git"
+  },
+  "homepage": "https://github.com/contextractor/contextractor",
+  "bugs": {
+    "url": "https://github.com/contextractor/contextractor/issues"
+  },
+  "keywords": [
+    "web-scraping",
+    "content-extraction",
+    "crawling",
+    "trafilatura",
+    "cli"
+  ],
+  "bin": {
+    "contextractor": "./cli.js"
+  },
+  "main": "./index.js",
+  "exports": {
+    ".": "./index.js",
+    "./cli": "./cli.js"
+  },
+  "files": [
+    "cli.js",
+    "index.js",
+    "postinstall.js"
+  ],
+  "scripts": {
+    "postinstall": "node postinstall.js"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ]
+}

package/postinstall.js

@@ -0,0 +1,97 @@
+"use strict";
+
+const https = require("https");
+const http = require("http");
+const fs = require("fs");
+const path = require("path");
+const { execSync } = require("child_process");
+const { getBinaryName } = require("./index");
+
+const REPO = "contextractor/contextractor";
+const BIN_DIR = path.join(__dirname, "bin");
+
+function getPackageVersion() {
+  const pkg = JSON.parse(
+    fs.readFileSync(path.join(__dirname, "package.json"), "utf8")
+  );
+  return pkg.version;
+}
+
+function follow(url) {
+  return new Promise((resolve, reject) => {
+    const mod = url.startsWith("https") ? https : http;
+    mod.get(url, { headers: { "User-Agent": "contextractor-npm" } }, (res) => {
+      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+        return follow(res.headers.location).then(resolve, reject);
+      }
+      if (res.statusCode !== 200) {
+        return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+      }
+      resolve(res);
+    }).on("error", reject);
+  });
+}
+
+async function download(url, dest) {
+  const res = await follow(url);
+  return new Promise((resolve, reject) => {
+    const file = fs.createWriteStream(dest);
+    res.pipe(file);
+    file.on("finish", () => file.close(resolve));
+    file.on("error", reject);
+  });
+}
+
+async function main() {
+  const binaryName = getBinaryName();
+  const version = getPackageVersion();
+  const tag = `v${version}`;
+  const url = `https://github.com/${REPO}/releases/download/${tag}/${binaryName}`;
+
+  fs.mkdirSync(BIN_DIR, { recursive: true });
+  const dest = path.join(BIN_DIR, binaryName);
+
+  // Skip if binary already exists
+  if (fs.existsSync(dest)) {
+    console.log(`Binary already exists: ${dest}`);
+    return;
+  }
+
+  console.log(`Downloading contextractor ${tag} for ${process.platform}-${process.arch}...`);
+  console.log(`  ${url}`);
+
+  try {
+    await download(url, dest);
+  } catch (err) {
+    console.error(
+      `Failed to download binary: ${err.message}\n\n` +
+        `Your platform (${process.platform}-${process.arch}) may not be supported.\n` +
+        `Supported: darwin-arm64, linux-x64, linux-arm64, win-x64\n` +
+        `See https://github.com/${REPO}/releases for available binaries.`
+    );
+    process.exit(1);
+  }
+
+  // Make executable on Unix
+  if (process.platform !== "win32") {
+    fs.chmodSync(dest, 0o755);
+  }
+
+  console.log("Binary installed successfully.");
+
+  // Install Playwright Chromium
+  console.log("Installing Playwright Chromium...");
+  try {
+    execSync("npx playwright install chromium", {
+      stdio: "inherit",
+      timeout: 300000,
+    });
+  } catch {
+    console.warn(
+      "Warning: Failed to install Playwright Chromium automatically.\n" +
+        "Run 'npx playwright install chromium' manually."
+    );
+  }
+}
+
+main();