@token-dashboard/typeless-usage-uploader 0.1.0 → 0.1.3

package/dist/collector.js

@@ -1,495 +0,0 @@
-import fs from 'node:fs';
-import os from 'node:os';
-import { DatabaseSync } from 'node:sqlite';
-import {
-  MAX_BATCH_BYTES,
-  MAX_EVENTS_PER_BATCH,
-  PERSISTENT_COLLECTOR_ID_PATH,
-  REGISTER_REFRESH_SECONDS,
-  TYPELESS_SCAN_PAGE_SIZE,
-} from './constants.js';
-import {
-  isoToEpochMs,
-  nowTs,
-  readPersistentCollectorId,
-  sha1,
-  sleep,
-  stableStringify,
-  writePersistentCollectorId,
-} from './utils.js';
-import { StateDb } from './state-db.js';
-
-const TYPELESS_TIMEZONE = 'Asia/Shanghai';
-
-export function computeTypelessRecordUid(typelessUserId, historyId) {
-  return sha1(`${typelessUserId}|${historyId}`);
-}
-
-export function loadTypelessAccount(storagePath) {
-  let payload;
-  try {
-    payload = JSON.parse(fs.readFileSync(storagePath, 'utf8'));
-  } catch {
-    return null;
-  }
-  const userData = payload?.userData;
-  if (!userData || typeof userData !== 'object') return null;
-  const role =
-    userData.role && typeof userData.role === 'object'
-      ? userData.role
-      : Array.isArray(userData.roles)
-        ? userData.roles[0]
-        : null;
-  return {
-    typelessUserId:
-      typeof userData.user_id === 'string' && userData.user_id.trim()
-        ? userData.user_id.trim()
-        : null,
-    typelessEmail:
-      typeof userData.email === 'string' && userData.email.trim()
-        ? userData.email.trim()
-        : null,
-    signupMethod:
-      typeof userData.signup_method === 'string' && userData.signup_method.trim()
-        ? userData.signup_method.trim()
-        : null,
-    typelessCreatedAt:
-      typeof userData.created_at === 'string' && userData.created_at.trim()
-        ? userData.created_at.trim()
-        : null,
-    typelessRole:
-      typeof role?.name === 'string' && role.name.trim() ? role.name.trim() : null,
-    typelessRoleExpiresAt:
-      typeof role?.exp_time === 'string' && role.exp_time.trim()
-        ? role.exp_time.trim()
-        : null,
-  };
-}
-
-export class TypelessUsageUploader {
-  constructor({
-    stateDbPath,
-    backendUrl,
-    typelessDbPath,
-    typelessStoragePath,
-    intervalSeconds,
-    persistentCollectorIdPath = PERSISTENT_COLLECTOR_ID_PATH,
-    pageSize = TYPELESS_SCAN_PAGE_SIZE,
-    timeZone = TYPELESS_TIMEZONE,
-  }) {
-    this.stateDb = new StateDb(stateDbPath);
-    this.backendUrl = backendUrl?.replace(/\/+$/, '') || null;
-    this.typelessDbPath = typelessDbPath;
-    this.typelessStoragePath = typelessStoragePath;
-    this.intervalSeconds = intervalSeconds;
-    this.persistentCollectorIdPath = persistentCollectorIdPath;
-    this.pageSize = pageSize;
-    this.timeZone = timeZone;
-    this.identity = this.ensureIdentity();
-  }
-
-  close() {
-    this.stateDb.close();
-  }
-
-  ensureIdentity() {
-    const identity = this.stateDb.getIdentity();
-    if (!identity.collectorId) {
-      const persisted = readPersistentCollectorId(this.persistentCollectorIdPath);
-      identity.collectorId = persisted ?? `${Date.now().toString(16)}${Math.random()
-        .toString(16)
-        .slice(2, 14)}`;
-    }
-    writePersistentCollectorId(this.persistentCollectorIdPath, identity.collectorId);
-    if (!identity.deviceId) {
-      identity.deviceId = `${os.hostname()}-${Math.random().toString(16).slice(2, 14)}`;
-    }
-    if (!identity.hostname) identity.hostname = os.hostname();
-    this.stateDb.setIdentity(identity);
-    return this.stateDb.getIdentity();
-  }
-
-  resetBackfillState() {
-    this.stateDb.resetBackfillState();
-  }
-
-  getQueueStats() {
-    return this.stateDb.getQueueStats();
-  }
-
-  emptyPayload(collector = null) {
-    return { collector, events: [] };
-  }
-
-  payloadHasData(payload) {
-    return Boolean(payload.events?.length);
-  }
-
-  payloadBytes(payload) {
-    return Buffer.byteLength(stableStringify(payload), 'utf8');
-  }
-
-  payloadCollectorKey(payload) {
-    return payload?.collector?.typelessUserId ?? '';
-  }
-
-  mergePayload(basePayload, incoming) {
-    const collector = basePayload.collector ?? incoming.collector ?? null;
-    const events = new Map(
-      (basePayload.events ?? []).map((item) => [item.recordUid, item]),
-    );
-    for (const item of incoming.events ?? []) events.set(item.recordUid, item);
-    return { collector, events: [...events.values()] };
-  }
-
-  shouldFlushChunk(payload) {
-    if (!this.payloadHasData(payload)) return false;
-    return (
-      payload.events.length >= MAX_EVENTS_PER_BATCH ||
-      this.payloadBytes(payload) >= MAX_BATCH_BYTES
-    );
-  }
-
-  wouldExceedUploadThreshold(payload) {
-    if (!this.payloadHasData(payload)) return false;
-    return (
-      payload.events.length > MAX_EVENTS_PER_BATCH ||
-      this.payloadBytes(payload) > MAX_BATCH_BYTES
-    );
-  }
-
-  appendPayloadToBuffer(incoming) {
-    if (!this.payloadHasData(incoming)) return 0;
-    const existing = this.stateDb.getBufferingBatch();
-    let queuedBatches = 0;
-    let payloadToSave = incoming;
-
-    if (existing) {
-      const basePayload = JSON.parse(existing.payload_json);
-      if (this.payloadCollectorKey(basePayload) !== this.payloadCollectorKey(incoming)) {
-        queuedBatches += this.stateDb.sealStaleBatches(true);
-      } else {
-        const merged = this.mergePayload(basePayload, incoming);
-        if (this.wouldExceedUploadThreshold(merged)) {
-          queuedBatches += this.stateDb.sealStaleBatches(true);
-        } else {
-          payloadToSave = merged;
-        }
-      }
-    }
-
-    this.stateDb.saveBufferingPayload(payloadToSave);
-    return queuedBatches + (this.stateDb.sealBufferIfThresholdHit() ? 1 : 0);
-  }
-
-  collectorRequestBody(account) {
-    return {
-      collectorId: this.identity.collectorId,
-      deviceId: this.identity.deviceId,
-      hostname: this.identity.hostname ?? os.hostname(),
-      typelessUserId: account.typelessUserId,
-      typelessEmail: account.typelessEmail,
-      signupMethod: account.signupMethod,
-      typelessCreatedAt: account.typelessCreatedAt,
-      typelessRole: account.typelessRole,
-      typelessRoleExpiresAt: account.typelessRoleExpiresAt,
-    };
-  }
-
-  async postJson(apiPath, payload) {
-    if (!this.backendUrl) throw new Error('backendUrl is not configured');
-    const response = await fetch(`${this.backendUrl}${apiPath}`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(payload),
-    });
-    if (!response.ok) {
-      const text = await response.text();
-      const detail = text ? `: ${text.slice(0, 500)}` : '';
-      throw new Error(`HTTP ${response.status} ${response.statusText}${detail}`);
-    }
-    const text = await response.text();
-    return text ? JSON.parse(text) : {};
-  }
-
-  openSourceDb() {
-    if (!fs.existsSync(this.typelessDbPath)) return null;
-    const db = new DatabaseSync(this.typelessDbPath, { readOnly: true });
-    db.exec('PRAGMA busy_timeout = 1000');
-    return db;
-  }
-
-  toEvent(account, row) {
-    const timestamp = isoToEpochMs(row.created_at);
-    if (timestamp == null) return null;
-    const recordUid = computeTypelessRecordUid(account.typelessUserId, row.id);
-    const charCount = Number(row.char_count ?? 0);
-    if (!Number.isFinite(charCount) || charCount <= 0) return null;
-    return {
-      recordUid,
-      timestamp,
-      timezone: this.timeZone,
-      mode: row.mode ?? null,
-      status: row.status ?? 'transcript',
-      appVersion: row.app_version ?? null,
-      charCount,
-      isDeleted: false,
-    };
-  }
-
-  eventFingerprint(event, updatedAt) {
-    return stableStringify({
-      timestamp: event.timestamp,
-      updatedAt: updatedAt ?? null,
-      status: event.status ?? null,
-      mode: event.mode ?? null,
-      appVersion: event.appVersion ?? null,
-      charCount: event.charCount,
-      isDeleted: Boolean(event.isDeleted),
-    });
-  }
-
-  buildTombstoneEvent(state) {
-    let previous;
-    try {
-      previous = JSON.parse(state.event_json || '{}');
-    } catch {
-      previous = {};
-    }
-    return {
-      recordUid: state.record_uid,
-      timestamp: Number(previous.timestamp ?? 0),
-      timezone: previous.timezone ?? this.timeZone,
-      mode: previous.mode ?? null,
-      status: previous.status ?? 'transcript',
-      appVersion: previous.appVersion ?? null,
-      charCount: 0,
-      isDeleted: true,
-    };
-  }
-
-  async scanTypeless() {
-    const result = {
-      accountFound: false,
-      dbFound: fs.existsSync(this.typelessDbPath),
-      scannedRecords: 0,
-      changedEvents: 0,
-      tombstoneEvents: 0,
-      batchesQueued: 0,
-    };
-    const account = loadTypelessAccount(this.typelessStoragePath);
-    if (!account?.typelessUserId) return result;
-    result.accountFound = true;
-    const db = this.openSourceDb();
-    if (!db) return result;
-
-    const existingRows = this.stateDb.getEventStates(account.typelessUserId);
-    const existingByUid = new Map(existingRows.map((row) => [row.record_uid, row]));
-    const seen = new Set();
-    const collector = this.collectorRequestBody(account);
-    let chunk = this.emptyPayload(collector);
-    let stateUpdates = [];
-
-    const flushChunk = () => {
-      if (!this.payloadHasData(chunk)) return;
-      result.batchesQueued += this.appendPayloadToBuffer(chunk);
-      this.stateDb.upsertEventStates(account.typelessUserId, stateUpdates);
-      chunk = this.emptyPayload(collector);
-      stateUpdates = [];
-    };
-
-    try {
-      const stmt = db.prepare(`
-        SELECT
-          id,
-          created_at,
-          updated_at,
-          status,
-          mode,
-          app_version,
-          LENGTH(refined_text) AS char_count
-        FROM history
-        WHERE user_id = ?
-          AND status = 'transcript'
-          AND refined_text IS NOT NULL
-          AND refined_text != ''
-        ORDER BY created_at ASC, id ASC
-        LIMIT ? OFFSET ?
-      `);
-
-      for (let offset = 0; ; offset += this.pageSize) {
-        const rows = stmt.all(account.typelessUserId, this.pageSize, offset);
-        if (!rows.length) break;
-        for (const row of rows) {
-          result.scannedRecords += 1;
-          const event = this.toEvent(account, row);
-          if (!event) continue;
-          seen.add(event.recordUid);
-          const fingerprint = this.eventFingerprint(event, row.updated_at);
-          const existing = existingByUid.get(event.recordUid);
-          if (
-            existing &&
-            existing.fingerprint === fingerprint &&
-            Number(existing.is_deleted) === 0
-          ) {
-            continue;
-          }
-          chunk = this.mergePayload(chunk, { events: [event] });
-          stateUpdates.push({
-            recordUid: event.recordUid,
-            fingerprint,
-            event,
-            isDeleted: false,
-          });
-          result.changedEvents += 1;
-          if (this.shouldFlushChunk(chunk)) flushChunk();
-        }
-      }
-
-      for (const state of existingRows) {
-        if (seen.has(state.record_uid) || Number(state.is_deleted) === 1) continue;
-        const event = this.buildTombstoneEvent(state);
-        const fingerprint = this.eventFingerprint(event, null);
-        chunk = this.mergePayload(chunk, { events: [event] });
-        stateUpdates.push({
-          recordUid: event.recordUid,
-          fingerprint,
-          event,
-          isDeleted: true,
-        });
-        result.tombstoneEvents += 1;
-        if (this.shouldFlushChunk(chunk)) flushChunk();
-      }
-
-      flushChunk();
-      return result;
-    } finally {
-      db.close();
-    }
-  }
-
-  async ensureRemoteRegistration(account) {
-    if (!this.backendUrl || !account?.typelessUserId) return;
-    const checkpointKey = `last_register_at:${account.typelessUserId}`;
-    const checkpoint = this.stateDb.getCheckpoint(checkpointKey);
-    if (checkpoint && nowTs() - Number(checkpoint) < REGISTER_REFRESH_SECONDS) return;
-    await this.postJson(
-      '/typeless-usage/collectors/register',
-      this.collectorRequestBody(account),
-    );
-    this.stateDb.setCheckpoint(checkpointKey, String(nowTs()));
-  }
-
-  async flushPendingBatches({ failFast = false, concurrency = 5 } = {}) {
-    if (!this.backendUrl) {
-      return { uploadedBatches: 0, failedBatches: 0, lastError: null };
-    }
-    const account = loadTypelessAccount(this.typelessStoragePath);
-    if (!account?.typelessUserId) {
-      return { uploadedBatches: 0, failedBatches: 0, lastError: null };
-    }
-
-    const fallbackCollectorBody = this.collectorRequestBody(account);
-    const rows = this.stateDb.iterPendingBatches();
-    if (!rows.length) {
-      try {
-        await this.ensureRemoteRegistration(account);
-      } catch (error) {
-        if (failFast) throw error;
-        return { uploadedBatches: 0, failedBatches: 1, lastError: error };
-      }
-      return { uploadedBatches: 0, failedBatches: 0, lastError: null };
-    }
-
-    let uploadedBatches = 0;
-    let failedBatches = 0;
-    let lastError = null;
-
-    const uploadOne = async (row) => {
-      const payload = JSON.parse(row.payload_json);
-      const collectorBody = payload.collector ?? fallbackCollectorBody;
-      if (!collectorBody?.typelessUserId) {
-        throw new Error('Typeless collector identity is missing from queued batch.');
-      }
-      await this.postJson('/typeless-usage/upload', {
-        idempotencyKey: row.batch_key,
-        collector: collectorBody,
-        payloadSizeBytes: Number(row.payload_bytes),
-        events: Array.isArray(payload.events) ? payload.events : [],
-      });
-    };
-
-    const pending = new Set();
-    let rowIndex = 0;
-    const enqueue = () => {
-      while (pending.size < concurrency && rowIndex < rows.length) {
-        const row = rows[rowIndex++];
-        const task = uploadOne(row)
-          .then(() => {
-            this.stateDb.markBatchUploaded(row.id);
-            uploadedBatches += 1;
-          })
-          .catch((error) => {
-            this.stateDb.markBatchFailed(
-              row.id,
-              Number(row.attempt_count) + 1,
-              error instanceof Error ? error.message : String(error),
-            );
-            failedBatches += 1;
-            lastError = error;
-            if (failFast) throw error;
-          })
-          .finally(() => pending.delete(task));
-        pending.add(task);
-      }
-    };
-
-    enqueue();
-    while (pending.size > 0) {
-      try {
-        await Promise.race(pending);
-      } catch {
-        if (failFast) break;
-      }
-      enqueue();
-    }
-
-    if (failFast && lastError) {
-      throw new Error(
-        `Typeless upload batch failed: ${
-          lastError instanceof Error ? lastError.message : String(lastError)
-        }`,
-      );
-    }
-
-    return { uploadedBatches, failedBatches, lastError };
-  }
-
-  async runCycle({ forceSeal = false } = {}) {
-    const scanResult = await this.scanTypeless();
-    scanResult.batchesQueued += this.stateDb.sealStaleBatches(forceSeal);
-    const flushResult = await this.flushPendingBatches();
-    const queueStats = this.getQueueStats();
-    return {
-      ...scanResult,
-      ...flushResult,
-      ...queueStats,
-      remainingQueuedBatches:
-        queueStats.bufferingBatchCount +
-        queueStats.pendingBatchCount +
-        queueStats.retryingBatchCount,
-    };
-  }
-
-  async watch() {
-    let firstCycle = true;
-    while (true) {
-      const result = await this.runCycle({ forceSeal: firstCycle });
-      console.log(
-        `[typeless] account=${result.accountFound ? 'yes' : 'no'} db=${result.dbFound ? 'yes' : 'no'} scanned=${result.scannedRecords} changed=${result.changedEvents} deleted=${result.tombstoneEvents} queued_batches=${result.batchesQueued} uploaded_batches=${result.uploadedBatches} remaining_batches=${result.remainingQueuedBatches}`,
-      );
-      firstCycle = false;
-      await sleep(this.intervalSeconds * 1000);
-    }
-  }
-}

package/dist/constants.js

@@ -1,43 +0,0 @@
-import os from 'node:os';
-import path from 'node:path';
-
-export const PRODUCT_NAME = 'Typeless 用量上报';
-export const CLI_NAME = 'typeless-usage-uploader';
-export const DEFAULT_TYPELESS_DB_PATH = path.join(
-  os.homedir(),
-  'Library',
-  'Application Support',
-  'Typeless',
-  'typeless.db',
-);
-export const DEFAULT_TYPELESS_STORAGE_PATH = path.join(
-  os.homedir(),
-  'Library',
-  'Application Support',
-  'Typeless',
-  'app-storage.json',
-);
-export const DEFAULT_INSTALL_ROOT = path.join(os.homedir(), '.typeless-usage-uploader');
-export const DEFAULT_CONFIG_FILE = path.join(DEFAULT_INSTALL_ROOT, 'config.json');
-export const DEFAULT_STATE_DB = path.join(DEFAULT_INSTALL_ROOT, 'state.sqlite');
-export const DEFAULT_LOG_DIR = path.join(DEFAULT_INSTALL_ROOT, 'logs');
-export const DEFAULT_STDOUT_LOG_PATH = path.join(DEFAULT_LOG_DIR, 'stdout.log');
-export const DEFAULT_STDERR_LOG_PATH = path.join(DEFAULT_LOG_DIR, 'stderr.log');
-export const DEFAULT_LAUNCHD_LABEL = 'com.token-dashboard.typeless-usage-uploader';
-export const DEFAULT_LAUNCH_AGENT_PATH = path.join(
-  os.homedir(),
-  'Library',
-  'LaunchAgents',
-  `${DEFAULT_LAUNCHD_LABEL}.plist`,
-);
-export const PERSISTENT_COLLECTOR_ID_PATH = path.join(
-  os.homedir(),
-  '.typeless-usage-uploader-collector-id',
-);
-export const DEFAULT_BACKEND_URL = 'http://101.126.66.51:8086';
-export const TYPELESS_SCAN_PAGE_SIZE = 500;
-export const MAX_EVENTS_PER_BATCH = 200;
-export const MAX_BATCH_BYTES = 1_000_000;
-export const MAX_BUFFER_AGE_SECONDS = 60;
-export const REGISTER_REFRESH_SECONDS = 600;
-export const STATUS_ONLINE_THRESHOLD_SECONDS = 600;

package/dist/launchd.js

@@ -1,154 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { spawnSync } from 'node:child_process';
-
-export function buildLaunchdPlist(runtime) {
-  const args = [
-    runtime.nodePath,
-    runtime.entryFile,
-    '--config-file',
-    runtime.configFile,
-    'run',
-  ];
-  return `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-  <dict>
-    <key>Label</key>
-    <string>${escapeXml(runtime.launchdLabel)}</string>
-    <key>ProgramArguments</key>
-    <array>
-${args.map((item) => `      <string>${escapeXml(item)}</string>`).join('\n')}
-    </array>
-    <key>RunAtLoad</key>
-    <true/>
-    <key>KeepAlive</key>
-    <true/>
-    <key>ProcessType</key>
-    <string>Background</string>
-    <key>WorkingDirectory</key>
-    <string>${escapeXml(runtime.installRoot)}</string>
-    <key>StandardOutPath</key>
-    <string>${escapeXml(runtime.stdoutLogPath)}</string>
-    <key>StandardErrorPath</key>
-    <string>${escapeXml(runtime.stderrLogPath)}</string>
-    <key>EnvironmentVariables</key>
-    <dict>
-      <key>NODE_NO_WARNINGS</key>
-      <string>1</string>
-    </dict>
-  </dict>
-</plist>
-`;
-}
-
-function escapeXml(value) {
-  return String(value)
-    .replaceAll('&', '&amp;')
-    .replaceAll('<', '&lt;')
-    .replaceAll('>', '&gt;')
-    .replaceAll('"', '&quot;')
-    .replaceAll("'", '&apos;');
-}
-
-export function parseLaunchctlPrint(output) {
-  const state = output.match(/^\s*state = ([^\n]+)$/m)?.[1]?.trim() ?? null;
-  const pidRaw = output.match(/^\s*pid = (\d+)$/m)?.[1] ?? null;
-  const exitRaw = output.match(/^\s*last exit code = (-?\d+)$/m)?.[1] ?? null;
-  return {
-    loaded: true,
-    running: state === 'running' || pidRaw != null,
-    pid: pidRaw ? Number(pidRaw) : null,
-    state,
-    lastExitCode: exitRaw ? Number(exitRaw) : null,
-  };
-}
-
-export class LaunchdServiceManager {
-  constructor(runtime) {
-    this.runtime = runtime;
-  }
-
-  ensureMacos() {
-    if (process.platform !== 'darwin') {
-      throw new Error('launchd management is only supported on macOS.');
-    }
-  }
-
-  domainTarget() {
-    return `gui/${process.getuid()}`;
-  }
-
-  serviceTarget() {
-    return `${this.domainTarget()}/${this.runtime.launchdLabel}`;
-  }
-
-  run(args, { check = true } = {}) {
-    const result = spawnSync(args[0], args.slice(1), { encoding: 'utf8' });
-    if (check && result.status !== 0) {
-      throw new Error(
-        result.stderr?.trim() || result.stdout?.trim() || `command failed: ${args.join(' ')}`,
-      );
-    }
-    return result;
-  }
-
-  ensurePlist() {
-    this.ensureMacos();
-    if (!this.runtime.entryFile || !fs.existsSync(this.runtime.entryFile)) {
-      throw new Error(`runtime entry not found: ${this.runtime.entryFile || '(empty)'}`);
-    }
-    const plist = buildLaunchdPlist(this.runtime);
-    fs.mkdirSync(path.dirname(this.runtime.plistPath), { recursive: true });
-    fs.mkdirSync(path.dirname(this.runtime.stdoutLogPath), { recursive: true });
-    fs.writeFileSync(this.runtime.plistPath, plist);
-    this.syncLaunchAgent(plist);
-  }
-
-  syncLaunchAgent(plist) {
-    if (!this.runtime.launchAgentPath) return;
-    if (this.runtime.autoStartOnLogin) {
-      fs.mkdirSync(path.dirname(this.runtime.launchAgentPath), { recursive: true });
-      fs.writeFileSync(this.runtime.launchAgentPath, plist);
-    } else if (fs.existsSync(this.runtime.launchAgentPath)) {
-      fs.rmSync(this.runtime.launchAgentPath, { force: true });
-    }
-  }
-
-  start() {
-    this.ensurePlist();
-    this.stop();
-    this.run(['launchctl', 'bootstrap', this.domainTarget(), this.runtime.plistPath]);
-  }
-
-  stop() {
-    this.ensureMacos();
-    this.run(['launchctl', 'bootout', this.serviceTarget()], { check: false });
-  }
-
-  restart() {
-    this.start();
-  }
-
-  status() {
-    this.ensureMacos();
-    const result = this.run(['launchctl', 'print', this.serviceTarget()], { check: false });
-    if (result.status !== 0) {
-      return {
-        loaded: false,
-        running: false,
-        pid: null,
-        state: null,
-        lastExitCode: null,
-      };
-    }
-    return parseLaunchctlPrint(result.stdout);
-  }
-
-  uninstall() {
-    this.stop();
-    for (const filePath of [this.runtime.plistPath, this.runtime.launchAgentPath]) {
-      if (filePath && fs.existsSync(filePath)) fs.unlinkSync(filePath);
-    }
-  }
-}