@nivalos/lithium.js 1.1.0 → 1.6.0

package/README.md

@@ -17,13 +17,14 @@ A flexible web proxy framework to make your skid dream a reality.
 - **`lithium doctor`**: one command to check your whole install
 - **Events + history**: `on("navigate", ...)`, `back()`/`forward()`/`reload()`/`search()`, works the same for every proxy
 - **Debug mode + network log**: `debug: true` for verbose tracing, `network_log()`/`on("request", ...)` for a devtools-lite view of the proxied page's `fetch`/XHR traffic
-- **Header policy**: block/allow specific headers, or bypass Lithium's interception entirely with `passthrough` mode
+- **Header policy**: block/allow/override specific headers, spoof device fingerprint (`navigator`/`screen`), or bypass interception entirely with `passthrough` mode
+- **Custom search engines**: register your own, with per-engine header overrides
 - **Modular design**: clean separation of client and server code, custom backends are first-class
 
 ## Installation
 
 ```bash
-npm install lithium.js
+npm install @nivalos/lithium.js
 ```
 
 Scramjet 2.x is published under the `alpha` tag and Lithium pins the exact
@@ -214,16 +215,40 @@ Lithium's Node server never sees the proxied site's raw HTTP (that goes browser
 ```javascript
 await init_lithium({
   headers: {
-    mode: "filter",              // "filter" (default) or "passthrough" (touch nothing)
-    block: ["x-frame-options"],  // stripped from requests AND responses, case-insensitive
-    allow: null,                 // if an array, ONLY these header names survive
+    mode: "filter",                    // "filter" (default) or "passthrough" (touch nothing)
+    block: ["x-frame-options"],        // stripped from requests AND responses, case-insensitive
+    allow: null,                       // if an array, ONLY these header names survive
+    set: { "x-custom": "value" },      // added/overridden on outgoing requests
+    searchEngineHeaders: { "x-a": "b" }, // extra `set` overrides, ONLY on requests to the current search engine's origin
+    referrer: "https://example.com/",  // passed to fetch()'s own `referrer` option (Referer can't be set via headers at all)
+    device: {                          // spoofs navigator/screen on the proxied page, see "Device spoofing" below
+      userAgent: "Mozilla/5.0 (...) Custom/1.0",
+      platform: "Win32",
+    },
   },
 })
-set_header_policy({ mode: "passthrough" })  // change it any time, no re-init needed
+set_header_policy({ mode: "passthrough" })  // change any of this any time, no re-init needed
 ```
 
 **Scope, honestly:** this rebuilds the `Response` object `fetch()` hands to the page, so it's real for anything the page's own JS reads. `XMLHttpRequest` traffic is logged but the policy isn't enforced on it. `<img>`, `<script src>`, `<link>`, and CSS loads never go through JS at all, so they're invisible to this — there's no hook point for them at this layer.
 
+**A real browser restriction, not a Lithium limitation:** a set of header names — `Cookie`, `Host`, `Origin`, `Connection`, anything starting with `Sec-`/`Proxy-`, and a few others — are "forbidden request headers" per the Fetch spec. No page JS anywhere can set them; the browser silently drops them when it builds the real request. `Referer` is on that list too, which is why it's handled separately via `referrer` above rather than through `set`/`block`/`allow` (put it in `set.referer` if that's more convenient — Lithium notices and routes it to the right place for you). `User-Agent` isn't spec-forbidden anymore, but Chrome still silently drops it from `fetch()` regardless (a long-standing Chromium quirk); Firefox honors it. `network_log()` entries include a `caveats` note whenever a header you configured falls into one of these categories, so you're not left guessing why it didn't take.
+
+### Device spoofing
+
+The `device` option overrides `navigator`/`screen` properties (`userAgent`, `platform`, `vendor`, `language`, `languages`, `maxTouchPoints`, `hardwareConcurrency`, `deviceMemory`, `userAgentData`, `screen: { width, height, ... }`) on the proxied page, so the *site's own JavaScript* — feature detection, analytics, fingerprinting — sees a different device. This is unrestricted (none of the forbidden-header rules above apply to JS properties), but it only fools client-side JS, not a server reading real HTTP headers, and only for reads that happen after this installs (same per-navigation timing caveat as everything else in this section).
+
+## Custom search engines
+
+```javascript
+import { search_engines } from "lithium.js" // or a client-only import in the browser
+
+search_engines.register("bing", (query) => `https://www.bing.com/search?q=${encodeURIComponent(query)}`)
+await init_lithium({ searchEngine: "bing" })   // or: config.searchEngine = "bing" any time
+```
+
+`search()`/`navigate()` with a non-URL query, and the `searchEngineHeaders` scoping above, all use whichever engine is currently set. An unregistered name silently falls back to Google rather than breaking navigation.
+
 ## Network log
 
 ```javascript

package/client/index.js

@@ -8,6 +8,8 @@
 //   navigate("example.com")
 
 import * as net from "./net.js"
+import { search_engines, build_search_url, search_origin } from "./search.js"
+export { search_engines }
 
 // the server injects this into your html (see server/index.js)
 const server_config = () => (typeof window !== "undefined" && window.__LITHIUM_CONFIG__) || {}
@@ -91,7 +93,9 @@ export const config = {
   set debug(on) { state.debug = Boolean(on) },
   // read-only snapshot (deep enough that mutating it can't affect live state);
   // use set_header_policy() to change it live
-  get headers() { return { ...state.header_policy, block: [...state.header_policy.block], allow: state.header_policy.allow ? [...state.header_policy.allow] : null } },
+  // read-only deep-enough snapshot (mutating it can't affect live state);
+  // use set_header_policy() to change it live
+  get headers() { return JSON.parse(JSON.stringify(state.header_policy)) },
 }
 
 // change the header policy without a full re-init (see net.js for what it can/can't do)
@@ -216,6 +220,7 @@ function get_iframe(create = true) {
             emit("request", entry)
           },
           dbg,
+          search_origin: () => search_origin(engine()),
         })
       } catch (err) {
         dbg("network hook install failed:", err)
@@ -346,9 +351,10 @@ export function is_url(v = "") {
   )
 }
 
+// engine can be "google", "duckduckgo", or anything registered via
+// search_engines.register()
 export function search_url(query, engine = "google") {
-  const q = encodeURIComponent(String(query).trim())
-  return engine === "duckduckgo" ? `https://duckduckgo.com/?q=${q}` : `https://www.google.com/search?q=${q}`
+  return build_search_url(query, engine)
 }
 
 // turn whatever was typed into a full url (or a search url)

package/client/net.js

@@ -5,9 +5,8 @@
 // real site, as bytes, not parsed HTTP). So this works client-side instead:
 // it's installed on the proxied iframe's window (same-origin, since that's
 // how UV/Scramjet serve proxied pages) and wraps fetch()/XMLHttpRequest.
-//   - fetch(): full control. Request headers are filtered before sending,
-//     and the Response is rebuilt with filtered headers before the page's
-//     own code sees it.
+//   - fetch(): full control over request headers going out and the Response
+//     the page's own code sees coming back.
 //   - XMLHttpRequest: observed only (logged), the header policy is NOT
 //     enforced on it — rebuilding a native XHR's headers isn't practical.
 //   - Anything that isn't fetch/XHR (<img>, <script src>, <link>, CSS,
@@ -15,15 +14,62 @@
 //   - Only requests made AFTER this installs are seen. It installs on the
 //     iframe's "load" event, so very early/synchronous requests on that
 //     page can be missed.
+//
+// A real browser restriction this can't route around: a set of header names
+// ("forbidden request headers" in the Fetch spec — Cookie, Host, Origin,
+// Connection, and anything starting with Sec-/Proxy-, among others) can't be
+// set from page JS at all; the browser silently drops them when it builds
+// the actual request, no matter what's in the Headers object handed to
+// fetch(). Referer is spec-forbidden the same way, but fetch() has a
+// dedicated `referrer` option that DOES work, so that one's routed there
+// instead. User-Agent is no longer spec-forbidden, but Chrome still silently
+// drops it from fetch() requests regardless (a long-standing Chromium
+// quirk); Firefox honors it. See is_unforgeable_header()/header_set_caveat().
+
+export const DEFAULT_POLICY = {
+  mode: "filter",
+  block: [],
+  allow: null,
+  set: {}, // headers to add/override on outgoing requests (subject to the browser restrictions above)
+  searchEngineHeaders: {}, // extra overrides, applied ONLY when the request's origin is the current search engine's
+  device: null, // { userAgent, platform, vendor, language, languages, maxTouchPoints, hardwareConcurrency, deviceMemory, userAgentData, screen } spoofed on navigator/screen
+  referrer: undefined, // passed straight to fetch()'s `referrer` option
+  referrerPolicy: undefined,
+}
+
+const UNFORGEABLE_HEADERS = new Set([
+  "accept-charset", "accept-encoding", "access-control-request-headers", "access-control-request-method",
+  "connection", "content-length", "cookie", "cookie2", "date", "dnt", "expect", "host", "keep-alive",
+  "origin", "permissions-policy", "referer", "set-cookie", "te", "trailer", "transfer-encoding", "upgrade", "via",
+])
+
+// true for header names the browser will silently refuse to send from page
+// JS no matter what Lithium does — see the file header for why
+export function is_unforgeable_header(name) {
+  const lk = String(name).toLowerCase()
+  return UNFORGEABLE_HEADERS.has(lk) || lk.startsWith("sec-") || lk.startsWith("proxy-")
+}
 
-export const DEFAULT_POLICY = { mode: "filter", block: [], allow: null }
+// a one-line explanation for a header name in a `set`/`searchEngineHeaders`
+// map, or null if there's nothing special to say about it
+export function header_set_caveat(name) {
+  const lk = String(name).toLowerCase()
+  if (lk === "user-agent") return "not spec-forbidden, but Chrome silently drops it from fetch() requests anyway (Chromium bug 571722); Firefox honors it"
+  if (lk === "referer" || lk === "referrer") return "Referer can't be set via headers at all; use policy.referrer instead (Lithium routes set.referer there for you)"
+  if (is_unforgeable_header(lk)) return "forbidden by the Fetch spec — the browser drops it silently, there's no workaround from page JS"
+  return null
+}
+
+// mode: "passthrough" -> do nothing (headers pass exactly as the proxy
+//       already delivered them, including ignoring set/searchEngineHeaders);
+//       "filter" -> apply block/allow, then (for request headers only) set/searchEngineHeaders
+// direction: "request" | "response" — set/searchEngineHeaders/referrer only make sense for
+//            outgoing requests (they're about how WE identify ourselves), not responses
+// search_engine_match: whether this request's origin is the current search engine's,
+//                       enabling searchEngineHeaders on top of set
+export function apply_policy(headers, policy, { direction = "request", search_engine_match = false } = {}) {
+  if (policy.mode === "passthrough") return { headers, blocked: [], overridden: [], caveats: [], referrer: undefined }
 
-// mode: "passthrough" -> do nothing (headers pass exactly as the proxy already
-//       delivered them); "filter" -> apply block/allow
-// block: header names (case-insensitive) to strip
-// allow: if an array, ONLY these header names survive (block still applies on top)
-export function apply_policy(headers, policy) {
-  if (policy.mode === "passthrough") return { headers, blocked: [] }
   const out = new Headers()
   const blocked = []
   const block = new Set((policy.block ?? []).map((h) => h.toLowerCase()))
@@ -36,19 +82,92 @@ export function apply_policy(headers, policy) {
     }
     out.append(k, v)
   }
-  return { headers: out, blocked }
+
+  if (direction !== "request") return { headers: out, blocked, overridden: [], caveats: [], referrer: undefined }
+
+  const overrides = { ...(policy.set ?? {}), ...(search_engine_match ? (policy.searchEngineHeaders ?? {}) : {}) }
+  const overridden = []
+  const caveats = []
+  let referrer = policy.referrer
+  for (const [name, value] of Object.entries(overrides)) {
+    const lk = name.toLowerCase()
+    if (lk === "referer" || lk === "referrer") {
+      referrer ??= value // an explicit policy.referrer still wins over one smuggled in via `set`
+      caveats.push({ name, note: header_set_caveat(name) })
+      continue // don't bother putting a header in that fetch() would drop anyway
+    }
+    out.set(name, value)
+    overridden.push(name)
+    const note = header_set_caveat(name)
+    if (note) caveats.push({ name, note })
+  }
+  return { headers: out, blocked, overridden, caveats, referrer }
+}
+
+// Overrides properties of navigator/screen on the proxied window so the
+// SITE'S OWN JS (feature detection, analytics, fingerprinting) sees a
+// different device. This is unrestricted (no Fetch-spec forbidden list to
+// fight), unlike request headers — but it only fools JS, not a server that
+// inspects real HTTP headers, and it only takes effect for code that reads
+// these properties AFTER this installs (see the file header).
+// Returns which property names were successfully overridden.
+export function apply_device_profile(win, device) {
+  if (!device || !win?.navigator) return []
+  const applied = []
+  const nav = win.navigator
+  const simple = ["userAgent", "platform", "vendor", "language", "languages", "maxTouchPoints", "hardwareConcurrency", "deviceMemory"]
+  for (const key of simple) {
+    if (device[key] === undefined) continue
+    try {
+      Object.defineProperty(nav, key, { get: () => device[key], configurable: true })
+      applied.push(key)
+    } catch {}
+  }
+  if (device.userAgentData) {
+    const uad = device.userAgentData
+    try {
+      Object.defineProperty(nav, "userAgentData", {
+        configurable: true,
+        get: () => ({
+          brands: uad.brands ?? [],
+          mobile: Boolean(uad.mobile),
+          platform: uad.platform ?? "",
+          toJSON: () => ({ brands: uad.brands ?? [], mobile: Boolean(uad.mobile), platform: uad.platform ?? "" }),
+          getHighEntropyValues: async (hints) =>
+            Object.fromEntries((hints ?? []).map((h) => [h, h in uad ? uad[h] : h === "brands" ? uad.brands ?? [] : h === "mobile" ? Boolean(uad.mobile) : h === "platform" ? uad.platform ?? "" : null])),
+        }),
+      })
+      applied.push("userAgentData")
+    } catch {}
+  }
+  if (device.screen && win.screen) {
+    for (const [key, value] of Object.entries(device.screen)) {
+      try {
+        Object.defineProperty(win.screen, key, { get: () => value, configurable: true })
+        applied.push(`screen.${key}`)
+      } catch {}
+    }
+  }
+  return applied
 }
 
 const headers_to_object = (headers) => Object.fromEntries(headers.entries())
 
 // win: the proxied iframe's contentWindow
-//   get_policy() -> current policy (read live, so changes apply mid-session)
-//   emit(entry)  -> called once per completed request
-//   dbg(...)     -> debug logger, called for policy decisions
-export function install(win, { get_policy, emit, dbg = () => {} }) {
+//   get_policy()    -> current policy (read live, so changes apply mid-session)
+//   emit(entry)     -> called once per completed request
+//   dbg(...)        -> debug logger
+//   search_origin() -> the current search engine's origin, or null (for searchEngineHeaders scoping)
+export function install(win, { get_policy, emit, dbg = () => {}, search_origin = () => null }) {
   if (!win || win.__lithiumNetHooked) return
   win.__lithiumNetHooked = true
 
+  const initial_policy = get_policy()
+  if (initial_policy.device) {
+    const applied = apply_device_profile(win, initial_policy.device)
+    dbg(`device profile applied: ${applied.length ? applied.join(", ") : "(nothing — properties weren't configurable here)"}`)
+  }
+
   const orig_fetch = win.fetch?.bind(win)
   if (orig_fetch) {
     win.fetch = async (input, init = {}) => {
@@ -56,16 +175,29 @@ export function install(win, { get_policy, emit, dbg = () => {} }) {
       const url = typeof input === "string" ? input : input.url
       const method = (init.method || (input instanceof Request ? input.method : "GET") || "GET").toUpperCase()
       const started = win.performance?.now?.() ?? Date.now()
+      const origin = search_origin()
+      const is_search_request = Boolean(origin) && url.startsWith(origin)
 
       let req_headers = new Headers(init.headers || (input instanceof Request ? input.headers : undefined) || {})
-      const { headers: filtered_req, blocked: blocked_req } = apply_policy(req_headers, policy)
-      if (blocked_req.length) dbg(`blocked request headers on ${url}: ${blocked_req.join(", ")}`)
+      const req_result = apply_policy(req_headers, policy, { direction: "request", search_engine_match: is_search_request })
+      req_headers = req_result.headers
+      if (req_result.blocked.length) dbg(`blocked request headers on ${url}: ${req_result.blocked.join(", ")}`)
+      for (const c of req_result.caveats) if (c.note) dbg(`request header "${c.name}" on ${url}: ${c.note}`)
+
+      const finish = (fields) =>
+        emit({
+          type: "fetch", method, url, ts: Date.now(), duration: Math.round((win.performance?.now?.() ?? Date.now()) - started),
+          requestHeaders: headers_to_object(req_headers), blockedRequestHeaders: req_result.blocked, blockedResponseHeaders: [], responseHeaders: null,
+          ...fields,
+        })
 
-      const finish = (fields) => emit({ type: "fetch", method, url, ts: Date.now(), duration: Math.round((win.performance?.now?.() ?? Date.now()) - started), requestHeaders: headers_to_object(filtered_req), blockedRequestHeaders: blocked_req, blockedResponseHeaders: [], responseHeaders: null, ...fields })
+      const fetch_init = { ...init, headers: req_headers }
+      if (req_result.referrer !== undefined) fetch_init.referrer = req_result.referrer
+      if (policy.referrerPolicy) fetch_init.referrerPolicy = policy.referrerPolicy
 
       let response
       try {
-        response = await orig_fetch(input instanceof Request ? new Request(input, { headers: filtered_req }) : input, { ...init, headers: filtered_req })
+        response = await orig_fetch(input instanceof Request ? new Request(input, { headers: req_headers }) : input, fetch_init)
       } catch (err) {
         finish({ status: 0, ok: false, error: err.message })
         throw err
@@ -75,12 +207,12 @@ export function install(win, { get_policy, emit, dbg = () => {} }) {
         finish({ status: response.status, ok: response.ok, responseHeaders: headers_to_object(response.headers) })
         return response
       }
-      const { headers: filtered_res, blocked: blocked_res } = apply_policy(response.headers, policy)
-      if (blocked_res.length) dbg(`blocked response headers on ${url}: ${blocked_res.join(", ")}`)
-      finish({ status: response.status, ok: response.ok, responseHeaders: headers_to_object(filtered_res), blockedResponseHeaders: blocked_res })
-      if (!blocked_res.length) return response
+      const res_result = apply_policy(response.headers, policy, { direction: "response" })
+      if (res_result.blocked.length) dbg(`blocked response headers on ${url}: ${res_result.blocked.join(", ")}`)
+      finish({ status: response.status, ok: response.ok, responseHeaders: headers_to_object(res_result.headers), blockedResponseHeaders: res_result.blocked })
+      if (!res_result.blocked.length) return response
       const body = await response.clone().blob()
-      return new Response(body, { status: response.status, statusText: response.statusText, headers: filtered_res })
+      return new Response(body, { status: response.status, statusText: response.statusText, headers: res_result.headers })
     }
   }
 

package/client/search.js

@@ -0,0 +1,37 @@
+// Search engine registry. Built-ins are "google" and "duckduckgo"; register
+// your own with a name and a function that turns a query into a URL.
+//   search_engines.register("bing", (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`)
+//   set_header_policy set the default via init_lithium({ searchEngine: "bing" })
+
+const engines = new Map([
+  ["google", (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`],
+  ["duckduckgo", (q) => `https://duckduckgo.com/?q=${encodeURIComponent(q)}`],
+])
+
+export const search_engines = {
+  list: () => [...engines.keys()],
+  has: (name) => engines.has(name),
+  register(name, url_builder) {
+    if (typeof name !== "string" || !name) throw new TypeError('search_engines.register(name, urlBuilder): name must be a non-empty string')
+    if (typeof url_builder !== "function") throw new TypeError(`search_engines.register("${name}", urlBuilder): urlBuilder must be a function (query) => url`)
+    engines.set(name, url_builder)
+  },
+}
+
+// falls back to "google" if the name isn't registered (never throws on lookup;
+// registering a bad one throws above, but using an unknown name shouldn't
+// break a running page over a typo)
+export function build_search_url(query, name) {
+  const build = engines.get(name) ?? engines.get("google")
+  return build(String(query).trim())
+}
+
+// the origin a search actually goes to, used to scope search-engine-only
+// header overrides (see net.js). null if the engine's URL is malformed.
+export function search_origin(name) {
+  try {
+    return new URL(build_search_url("x", name)).origin
+  } catch {
+    return null
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nivalos/lithium.js",
-  "version": "1.1.0",
+  "version": "1.6.0",
   "description": "Creating a proxy has never been easier with Lithium.js",
   "type": "module",
   "main": "server/index.js",
@@ -10,6 +10,13 @@
   "exports": {
     ".": "./server/index.js"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Lithium-Foundry/Lithium.js.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
   "scripts": {
     "start": "node server/start.js",
     "test": "echo \"no automated tests are bundled, run `npx lithium doctor` to check your install\""