@rester159/blacktip 0.1.0 → 0.4.0

package/native/tls-client/main.go

@@ -0,0 +1,216 @@
+// BlackTip TLS side-channel daemon.
+//
+// Reads newline-delimited JSON requests from stdin, performs HTTP
+// requests with a real Chrome TLS fingerprint via bogdanfinn/tls-client,
+// and writes newline-delimited JSON responses to stdout.
+//
+// This is the v0.3.0 answer to the question: "what do I do when an edge
+// gates the very first request before BlackTip's browser has a session?"
+// You use this daemon to make the gating request — it presents a real
+// Chrome TLS ClientHello, real H2 frames, and real headers — capture
+// the cookies and tokens it gets back, and inject them into the
+// browser session before the user-driven flow continues.
+//
+// Wire format:
+//
+//   Request:  {"id":"<string>","url":"<string>","method":"<GET|POST|...>","headers":{"...":"..."},"body":"<string base64>","timeoutMs":15000,"profile":"chrome_133"}
+//   Response: {"id":"<string>","ok":true,"status":200,"headers":{"...":["...","..."]},"body":"<string base64>","finalUrl":"<string>","durationMs":123}
+//             OR
+//             {"id":"<string>","ok":false,"error":"<message>","durationMs":123}
+//
+// One JSON object per line in both directions. The Node parent reads
+// stdout line-by-line and matches responses by id. The daemon stays
+// alive across many requests so we don't pay subprocess startup cost
+// per call.
+package main
+
+import (
+	"bufio"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	http "github.com/bogdanfinn/fhttp"
+	tls_client "github.com/bogdanfinn/tls-client"
+	"github.com/bogdanfinn/tls-client/profiles"
+)
+
+type request struct {
+	ID        string            `json:"id"`
+	URL       string            `json:"url"`
+	Method    string            `json:"method"`
+	Headers   map[string]string `json:"headers"`
+	Body      string            `json:"body"`
+	TimeoutMs int               `json:"timeoutMs"`
+	Profile   string            `json:"profile"`
+}
+
+type response struct {
+	ID         string              `json:"id"`
+	OK         bool                `json:"ok"`
+	Status     int                 `json:"status,omitempty"`
+	Headers    map[string][]string `json:"headers,omitempty"`
+	Body       string              `json:"body,omitempty"`
+	FinalURL   string              `json:"finalUrl,omitempty"`
+	DurationMs int64               `json:"durationMs"`
+	Error      string              `json:"error,omitempty"`
+}
+
+// resolveProfile maps a profile name to a tls-client ClientProfile.
+// Defaults to the latest Chrome at the time of writing.
+func resolveProfile(name string) profiles.ClientProfile {
+	switch strings.ToLower(name) {
+	case "chrome_120":
+		return profiles.Chrome_120
+	case "chrome_124":
+		return profiles.Chrome_124
+	case "chrome_131":
+		return profiles.Chrome_131
+	case "chrome_133":
+		return profiles.Chrome_133
+	case "firefox_120":
+		return profiles.Firefox_120
+	case "safari_ios_16_0":
+		return profiles.Safari_IOS_16_0
+	default:
+		return profiles.Chrome_133
+	}
+}
+
+// buildClient constructs a tls-client with the requested profile and timeout.
+// We rebuild on every request because timeout is per-request and the cost
+// is negligible compared to the network round-trip.
+func buildClient(profile profiles.ClientProfile, timeoutMs int) (tls_client.HttpClient, error) {
+	if timeoutMs <= 0 {
+		timeoutMs = 15000
+	}
+	options := []tls_client.HttpClientOption{
+		tls_client.WithTimeoutSeconds(timeoutMs / 1000),
+		tls_client.WithClientProfile(profile),
+		tls_client.WithNotFollowRedirects(),
+	}
+	return tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
+}
+
+func handle(req request) response {
+	start := time.Now()
+	durationFor := func() int64 { return time.Since(start).Milliseconds() }
+
+	if req.URL == "" {
+		return response{ID: req.ID, OK: false, Error: "url is required", DurationMs: durationFor()}
+	}
+	method := req.Method
+	if method == "" {
+		method = "GET"
+	}
+
+	client, err := buildClient(resolveProfile(req.Profile), req.TimeoutMs)
+	if err != nil {
+		return response{ID: req.ID, OK: false, Error: "buildClient: " + err.Error(), DurationMs: durationFor()}
+	}
+
+	var bodyReader io.Reader
+	if req.Body != "" {
+		decoded, decErr := base64.StdEncoding.DecodeString(req.Body)
+		if decErr != nil {
+			return response{ID: req.ID, OK: false, Error: "body base64 decode: " + decErr.Error(), DurationMs: durationFor()}
+		}
+		bodyReader = strings.NewReader(string(decoded))
+	}
+
+	httpReq, err := http.NewRequest(method, req.URL, bodyReader)
+	if err != nil {
+		return response{ID: req.ID, OK: false, Error: "NewRequest: " + err.Error(), DurationMs: durationFor()}
+	}
+
+	for k, v := range req.Headers {
+		httpReq.Header.Set(k, v)
+	}
+	// If the caller didn't set Accept-Language, fall back to a Chrome default.
+	if httpReq.Header.Get("Accept-Language") == "" {
+		httpReq.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	}
+
+	resp, err := client.Do(httpReq)
+	if err != nil {
+		return response{ID: req.ID, OK: false, Error: "Do: " + err.Error(), DurationMs: durationFor()}
+	}
+	defer resp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return response{ID: req.ID, OK: false, Error: "read body: " + err.Error(), DurationMs: durationFor()}
+	}
+
+	headers := make(map[string][]string, len(resp.Header))
+	for k, v := range resp.Header {
+		headers[k] = v
+	}
+
+	finalURL := req.URL
+	if resp.Request != nil && resp.Request.URL != nil {
+		finalURL = resp.Request.URL.String()
+	}
+
+	return response{
+		ID:         req.ID,
+		OK:         true,
+		Status:     resp.StatusCode,
+		Headers:    headers,
+		Body:       base64.StdEncoding.EncodeToString(bodyBytes),
+		FinalURL:   finalURL,
+		DurationMs: durationFor(),
+	}
+}
+
+func main() {
+	scanner := bufio.NewScanner(os.Stdin)
+	// Allow large request bodies (e.g. POSTed forms with file fields).
+	scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
+
+	// Stdout writes are mutex-protected so concurrent handlers don't
+	// interleave their JSON lines.
+	var stdoutMu sync.Mutex
+	emit := func(r response) {
+		out, err := json.Marshal(r)
+		if err != nil {
+			out = []byte(fmt.Sprintf(`{"id":%q,"ok":false,"error":"marshal failed: %s"}`, r.ID, err.Error()))
+		}
+		stdoutMu.Lock()
+		defer stdoutMu.Unlock()
+		os.Stdout.Write(out)
+		os.Stdout.Write([]byte("\n"))
+	}
+
+	var wg sync.WaitGroup
+	for scanner.Scan() {
+		line := append([]byte(nil), scanner.Bytes()...)
+		if len(line) == 0 {
+			continue
+		}
+		var req request
+		if err := json.Unmarshal(line, &req); err != nil {
+			emit(response{ID: "", OK: false, Error: "unmarshal: " + err.Error()})
+			continue
+		}
+		// Handle requests concurrently — the Go TLS client is goroutine-safe.
+		wg.Add(1)
+		go func(r request) {
+			defer wg.Done()
+			emit(handle(r))
+		}(req)
+	}
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "scan error:", err)
+		os.Exit(1)
+	}
+	// Wait for any in-flight requests to drain before exiting. Without
+	// this, closing stdin races the goroutines and the parent never
+	// sees the response.
+	wg.Wait()
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rester159/blacktip",
-  "version": "0.1.0",
+  "version": "0.4.0",
   "description": "Stealth browser instrument for AI agents. Real Chrome + patchright CDP stealth + human-calibrated behavioral simulation. Every action is indistinguishable from a human.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,8 +21,14 @@
     "dist",
     "README.md",
     "AGENTS.md",
+    "CHANGELOG.md",
     "LICENSE",
-    "examples"
+    "examples",
+    "docs",
+    "native/tls-client/main.go",
+    "native/tls-client/go.mod",
+    "native/tls-client/go.sum",
+    "scripts"
   ],
   "scripts": {
     "build": "tsc",

package/scripts/fit-cmu-keystroke.mjs

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+/**
+ * Fit a behavioral profile against the real CMU Keystroke Dynamics
+ * dataset and report a held-out validation result.
+ *
+ * This is the v0.3.0 calibration validation script. It:
+ *
+ *   1. Loads `data/cmu-keystroke/DSL-StrongPasswordData.csv` (51 subjects
+ *      × 8 sessions × 50 reps = 20,400 phrases of `.tie5Roanl`).
+ *   2. Splits subjects 80/20 into training and held-out sets.
+ *   3. Fits a `CalibratedProfile` against the training subjects.
+ *   4. Reports the fitted distribution parameters.
+ *   5. Computes a Kolmogorov–Smirnov-style distribution-similarity score
+ *      (max CDF distance) of the fit against (a) the training set,
+ *      (b) the held-out set, and (c) BlackTip's canonical HUMAN_PROFILE.
+ *      A good fit beats the canonical profile on the held-out set.
+ *   6. Writes the resulting profile to
+ *      `data/cmu-keystroke/calibrated-profile.json` so users can load
+ *      it in their own code without re-running the fit every time.
+ *
+ * Run with: node scripts/fit-cmu-keystroke.mjs
+ */
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseCmuKeystrokeCsv } from '../dist/behavioral/parsers.js';
+import { fitTypingDynamics, fitMouseDynamics, deriveProfileConfig } from '../dist/behavioral/calibration.js';
+import { HUMAN_PROFILE } from '../dist/behavioral-engine.js';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..');
+const csvPath = join(root, 'data', 'cmu-keystroke', 'DSL-StrongPasswordData.csv');
+const outPath = join(root, 'data', 'cmu-keystroke', 'calibrated-profile.json');
+
+console.log('Loading CMU CSV from', csvPath);
+const csv = readFileSync(csvPath, 'utf-8');
+const allSessions = parseCmuKeystrokeCsv(csv);
+console.log(`Parsed ${allSessions.length} typing sessions (each is one rep of .tie5Roanl)`);
+
+if (allSessions.length === 0) {
+  console.error('No sessions parsed — check CSV format');
+  process.exit(1);
+}
+
+// The phrase has 11 keys; sanity check the first session.
+const first = allSessions[0];
+console.log(`First session: ${first.keystrokes.length} keystrokes, phrase=${first.phrase}`);
+if (first.keystrokes.length !== 11) {
+  console.error(`Expected 11 keystrokes per session, got ${first.keystrokes.length}`);
+  process.exit(1);
+}
+
+// CMU subjects are encoded in the source CSV but not exposed by the
+// parser (which throws away subject IDs). For an honest 80/20 train/test
+// split we re-read the CSV's first column ourselves.
+const lines = csv.trim().split(/\r?\n/);
+const subjectsInOrder = lines.slice(1).map((l) => l.split(',')[0]);
+const uniqueSubjects = [...new Set(subjectsInOrder)];
+console.log(`Found ${uniqueSubjects.length} unique subjects`);
+
+// Deterministic 80/20 split — sort then slice, so re-runs are stable.
+const trainSubjects = new Set(uniqueSubjects.slice(0, Math.floor(uniqueSubjects.length * 0.8)));
+const testSubjects = new Set(uniqueSubjects.slice(Math.floor(uniqueSubjects.length * 0.8)));
+console.log(`Train: ${trainSubjects.size} subjects, Test: ${testSubjects.size} subjects`);
+
+const trainSessions = [];
+const testSessions = [];
+for (let i = 0; i < allSessions.length; i++) {
+  const subj = subjectsInOrder[i];
+  if (trainSubjects.has(subj)) trainSessions.push(allSessions[i]);
+  else if (testSubjects.has(subj)) testSessions.push(allSessions[i]);
+}
+console.log(`Train sessions: ${trainSessions.length}, Test sessions: ${testSessions.length}`);
+
+// Fit
+console.log('\nFitting typing dynamics on training set...');
+const trainTyping = fitTypingDynamics(trainSessions);
+console.log(`  Hold time:   mean=${trainTyping.holdTime.mean.toFixed(1)}ms  p5=${trainTyping.holdTime.p5.toFixed(1)}  p50=${trainTyping.holdTime.p50.toFixed(1)}  p95=${trainTyping.holdTime.p95.toFixed(1)}`);
+console.log(`  Flight time: mean=${trainTyping.flightTime.mean.toFixed(1)}ms  p5=${trainTyping.flightTime.p5.toFixed(1)}  p50=${trainTyping.flightTime.p50.toFixed(1)}  p95=${trainTyping.flightTime.p95.toFixed(1)}`);
+console.log(`  Digraphs fit: ${Object.keys(trainTyping.digraphFlightTime).length}`);
+console.log(`  Mistake rate: ${(trainTyping.mistakeRate * 100).toFixed(2)}%`);
+
+// Mouse fit isn't applicable to CMU keystroke data — leave it at the
+// canonical defaults so the derived ProfileConfig is well-formed.
+const mouseFit = fitMouseDynamics([]);
+
+// Held-out evaluation: extract raw flight and hold samples from each set,
+// then compare empirical CDFs.
+const flightsFromSessions = (sessions) => {
+  const out = [];
+  for (const s of sessions) for (let i = 1; i < s.keystrokes.length; i++) out.push(s.keystrokes[i].flightTimeMs);
+  return out;
+};
+const holdsFromSessions = (sessions) => {
+  const out = [];
+  for (const s of sessions) for (const k of s.keystrokes) out.push(k.holdTimeMs);
+  return out;
+};
+
+const ksDistance = (a, b) => {
+  // Empirical KS distance: max |F_a(x) - F_b(x)| over the merged sample set.
+  const sortedA = [...a].sort((x, y) => x - y);
+  const sortedB = [...b].sort((x, y) => x - y);
+  const all = [...new Set([...sortedA, ...sortedB])].sort((x, y) => x - y);
+  const cdf = (sorted, x) => {
+    let lo = 0, hi = sorted.length;
+    while (lo < hi) { const mid = (lo + hi) >> 1; if (sorted[mid] <= x) lo = mid + 1; else hi = mid; }
+    return lo / sorted.length;
+  };
+  let maxDiff = 0;
+  for (const x of all) {
+    const d = Math.abs(cdf(sortedA, x) - cdf(sortedB, x));
+    if (d > maxDiff) maxDiff = d;
+  }
+  return maxDiff;
+};
+
+const trainFlights = flightsFromSessions(trainSessions);
+const testFlights = flightsFromSessions(testSessions);
+const trainHolds = holdsFromSessions(trainSessions);
+const testHolds = holdsFromSessions(testSessions);
+
+// Synthesize samples from BlackTip's canonical profile to compare against.
+// HUMAN_PROFILE.typingSpeedMs is a [min, max] uniform-ish range — sample
+// 5000 values uniformly to build a synthetic distribution.
+const sampleUniform = (lo, hi, n) => {
+  const out = [];
+  for (let i = 0; i < n; i++) out.push(lo + Math.random() * (hi - lo));
+  return out;
+};
+const canonicalFlights = sampleUniform(HUMAN_PROFILE.typingSpeedMs[0], HUMAN_PROFILE.typingSpeedMs[1], 5000);
+const canonicalHolds = sampleUniform(HUMAN_PROFILE.clickDwellMs?.[0] ?? 40, HUMAN_PROFILE.clickDwellMs?.[1] ?? 100, 5000);
+
+// Synthesize "calibrated" samples from the fitted distribution by
+// sampling within [p5, p95]. This mirrors what BehavioralEngine will
+// actually emit when configured with the fitted ProfileConfig.
+const calibratedFlights = sampleUniform(trainTyping.flightTime.p5, trainTyping.flightTime.p95, 5000);
+const calibratedHolds = sampleUniform(trainTyping.holdTime.p5, trainTyping.holdTime.p95, 5000);
+
+const ksCanonicalFlight = ksDistance(testFlights, canonicalFlights);
+const ksCalibratedFlight = ksDistance(testFlights, calibratedFlights);
+const ksCanonicalHold = ksDistance(testHolds, canonicalHolds);
+const ksCalibratedHold = ksDistance(testHolds, calibratedHolds);
+
+console.log('\n=== Held-out KS distance (lower = closer to real human distribution) ===');
+console.log(`Flight time:`);
+console.log(`  Canonical HUMAN_PROFILE [${HUMAN_PROFILE.typingSpeedMs[0]}, ${HUMAN_PROFILE.typingSpeedMs[1]}]ms:  KS=${ksCanonicalFlight.toFixed(4)}`);
+console.log(`  Calibrated [p5=${trainTyping.flightTime.p5.toFixed(0)}, p95=${trainTyping.flightTime.p95.toFixed(0)}]ms:        KS=${ksCalibratedFlight.toFixed(4)}`);
+console.log(`  Improvement: ${(ksCanonicalFlight - ksCalibratedFlight).toFixed(4)} (${((1 - ksCalibratedFlight / ksCanonicalFlight) * 100).toFixed(1)}% closer)`);
+console.log(`Hold time:`);
+console.log(`  Canonical clickDwellMs [${HUMAN_PROFILE.clickDwellMs?.[0]}, ${HUMAN_PROFILE.clickDwellMs?.[1]}]ms:  KS=${ksCanonicalHold.toFixed(4)}`);
+console.log(`  Calibrated [p5=${trainTyping.holdTime.p5.toFixed(0)}, p95=${trainTyping.holdTime.p95.toFixed(0)}]ms:           KS=${ksCalibratedHold.toFixed(4)}`);
+console.log(`  Improvement: ${(ksCanonicalHold - ksCalibratedHold).toFixed(4)} (${((1 - ksCalibratedHold / ksCanonicalHold) * 100).toFixed(1)}% closer)`);
+
+const profileConfig = deriveProfileConfig(mouseFit, trainTyping);
+const calibrated = {
+  name: 'cmu-keystroke-2009',
+  source: 'CMU Keystroke Dynamics dataset (Killourhy & Maxion 2009), 80% subject train split',
+  fittedAt: new Date().toISOString(),
+  trainSubjects: trainSubjects.size,
+  testSubjects: testSubjects.size,
+  trainSessions: trainSessions.length,
+  testSessions: testSessions.length,
+  validation: {
+    flightTime: {
+      canonicalKsDistance: ksCanonicalFlight,
+      calibratedKsDistance: ksCalibratedFlight,
+      improvementRatio: 1 - ksCalibratedFlight / ksCanonicalFlight,
+    },
+    holdTime: {
+      canonicalKsDistance: ksCanonicalHold,
+      calibratedKsDistance: ksCalibratedHold,
+      improvementRatio: 1 - ksCalibratedHold / ksCanonicalHold,
+    },
+  },
+  fits: {
+    typing: trainTyping,
+  },
+  profileConfig,
+};
+
+writeFileSync(outPath, JSON.stringify(calibrated, null, 2));
+console.log(`\nWrote calibrated profile to ${outPath}`);
+console.log(`\nUse it via:`);
+console.log(`  import calibrated from './data/cmu-keystroke/calibrated-profile.json' with { type: 'json' };`);
+console.log(`  const bt = new BlackTip({ behaviorProfile: calibrated.profileConfig });`);