@totalreclaw/totalreclaw 1.0.4 → 1.1.0

package/subgraph-search.ts

@@ -0,0 +1,282 @@
+/**
+ * Subgraph search path — queries facts via GraphQL hash_in.
+ *
+ * Used when the managed service is active (TOTALRECLAW_SELF_HOSTED is not
+ * "true"). Replaces the HTTP POST to /v1/search with a GraphQL query to
+ * the subgraph via the relay server.
+ *
+ * The relay server proxies GraphQL queries to Graph Studio with its own
+ * API key at `${relayUrl}/v1/subgraph`. Clients never need a subgraph endpoint.
+ *
+ * Query cost optimization:
+ *   Phase 1: Single query with ALL trapdoors (1 query).
+ *   Phase 2: If saturated (1000 results), split into small parallel batches
+ *            so rare trapdoor matches aren't drowned by common ones.
+ *   Phase 3: Cursor-based pagination for any saturated batch.
+ *
+ * This minimizes Graph Network query costs (pay-per-query via GRT):
+ *   - Small datasets (<1000 matches): 1 query total
+ *   - Medium datasets: 1 + N batch queries
+ *   - Large datasets: 1 + N batches + pagination queries
+ */
+
+import { getSubgraphConfig } from './subgraph-store.js';
+
+export interface SubgraphSearchFact {
+  id: string;
+  encryptedBlob: string;
+  encryptedEmbedding: string | null;
+  decayScore: string;
+  timestamp: string;
+  isActive: boolean;
+}
+
+/** Batch size for Phase 2 split queries. */
+const TRAPDOOR_BATCH_SIZE = parseInt(process.env.TOTALRECLAW_TRAPDOOR_BATCH_SIZE ?? '5', 10);
+/** Graph Studio / Graph Network hard limit on `first` argument. */
+const PAGE_SIZE = parseInt(process.env.TOTALRECLAW_SUBGRAPH_PAGE_SIZE ?? '1000', 10);
+
+/**
+ * Execute a single GraphQL query against the subgraph endpoint.
+ * Returns null on any network or HTTP error (never throws).
+ */
+async function gqlQuery<T>(
+  endpoint: string,
+  query: string,
+  variables: Record<string, unknown>,
+  authKeyHex?: string,
+): Promise<T | null> {
+  try {
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'X-TotalReclaw-Client': 'openclaw-plugin',
+    };
+    if (authKeyHex) {
+      headers['Authorization'] = `Bearer ${authKeyHex}`;
+    }
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({ query, variables }),
+    });
+    if (!response.ok) return null;
+    const json = await response.json() as { data?: T };
+    return json.data ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/** GraphQL query for blind index lookup. */
+const SEARCH_QUERY = `
+  query SearchByBlindIndex($trapdoors: [String!]!, $owner: Bytes!, $first: Int!) {
+    blindIndexes(
+      where: { hash_in: $trapdoors, owner: $owner, fact_: { isActive: true } }
+      first: $first
+      orderBy: id
+      orderDirection: desc
+    ) {
+      id
+      fact {
+        id
+        encryptedBlob
+        encryptedEmbedding
+        decayScore
+        timestamp
+        isActive
+        contentFp
+        sequenceId
+        version
+      }
+    }
+  }
+`;
+
+/** Pagination query — cursor-based using id_gt, ascending for deterministic walk. */
+const PAGINATE_QUERY = `
+  query PaginateBlindIndex($trapdoors: [String!]!, $owner: Bytes!, $first: Int!, $lastId: String!) {
+    blindIndexes(
+      where: { hash_in: $trapdoors, owner: $owner, id_gt: $lastId, fact_: { isActive: true } }
+      first: $first
+      orderBy: id
+      orderDirection: asc
+    ) {
+      id
+      fact {
+        id
+        encryptedBlob
+        encryptedEmbedding
+        timestamp
+        decayScore
+        isActive
+        contentFp
+        sequenceId
+        version
+      }
+    }
+  }
+`;
+
+interface BlindIndexEntry {
+  id: string;
+  fact: SubgraphSearchFact;
+}
+
+interface SearchResponse {
+  blindIndexes?: BlindIndexEntry[];
+}
+
+/** Collect facts from blind index entries, deduplicating by fact id. */
+function collectFacts(
+  entries: BlindIndexEntry[],
+  allResults: Map<string, SubgraphSearchFact>,
+): void {
+  for (const entry of entries) {
+    if (entry.fact && entry.fact.isActive !== false && !allResults.has(entry.fact.id)) {
+      allResults.set(entry.fact.id, entry.fact);
+    }
+  }
+}
+
+/**
+ * Paginate a single trapdoor chunk until exhausted or maxCandidates reached.
+ */
+async function paginateChunk(
+  subgraphUrl: string,
+  chunk: string[],
+  owner: string,
+  allResults: Map<string, SubgraphSearchFact>,
+  maxCandidates: number,
+  authKeyHex?: string,
+): Promise<void> {
+  let lastId = '';
+  while (allResults.size < maxCandidates) {
+    const data = await gqlQuery<SearchResponse>(
+      subgraphUrl,
+      PAGINATE_QUERY,
+      { trapdoors: chunk, owner, first: PAGE_SIZE, lastId },
+      authKeyHex,
+    );
+    const entries = data?.blindIndexes ?? [];
+    if (entries.length === 0) break;
+    collectFacts(entries, allResults);
+    if (entries.length < PAGE_SIZE) break;
+    lastId = entries[entries.length - 1].id;
+  }
+}
+
+/**
+ * Search the subgraph for facts matching the given trapdoors.
+ *
+ * Adaptive strategy to minimize query costs:
+ *
+ *   Phase 1: Single query with ALL trapdoors.
+ *     - If not saturated (< PAGE_SIZE results): done. 1 query total.
+ *     - If saturated: common trapdoors may be drowning rare ones. Go to Phase 2.
+ *
+ *   Phase 2: Split trapdoors into small parallel batches (TRAPDOOR_BATCH_SIZE=5).
+ *     - Each batch independently gets up to PAGE_SIZE results.
+ *     - Rare trapdoor matches get their own budget.
+ *
+ *   Phase 3: Cursor-based pagination for any saturated batch.
+ *     - Only for power users with very large datasets.
+ */
+export async function searchSubgraph(
+  owner: string,
+  trapdoors: string[],
+  maxCandidates: number,
+  authKeyHex?: string,
+): Promise<SubgraphSearchFact[]> {
+  const config = getSubgraphConfig();
+  const subgraphUrl = `${config.relayUrl}/v1/subgraph`;
+  const allResults = new Map<string, SubgraphSearchFact>();
+
+  // -----------------------------------------------------------------------
+  // Phase 1: Single query with all trapdoors (1 query)
+  // -----------------------------------------------------------------------
+  const phase1 = await gqlQuery<SearchResponse>(
+    subgraphUrl,
+    SEARCH_QUERY,
+    { trapdoors, owner, first: PAGE_SIZE },
+    authKeyHex,
+  );
+
+  const phase1Entries = phase1?.blindIndexes ?? [];
+  collectFacts(phase1Entries, allResults);
+
+  // Not saturated — we got everything in 1 query. Done.
+  if (phase1Entries.length < PAGE_SIZE) {
+    return Array.from(allResults.values());
+  }
+
+  // -----------------------------------------------------------------------
+  // Phase 2: Saturated — split into small batches for better rare-word recall.
+  // Common trapdoors were drowning rare ones in the single-query result.
+  // -----------------------------------------------------------------------
+  const chunks: string[][] = [];
+  for (let i = 0; i < trapdoors.length; i += TRAPDOOR_BATCH_SIZE) {
+    chunks.push(trapdoors.slice(i, i + TRAPDOOR_BATCH_SIZE));
+  }
+
+  const batchResults = await Promise.all(
+    chunks.map(async (chunk) => {
+      const data = await gqlQuery<SearchResponse>(
+        subgraphUrl,
+        SEARCH_QUERY,
+        { trapdoors: chunk, owner, first: PAGE_SIZE },
+        authKeyHex,
+      );
+      return { chunk, entries: data?.blindIndexes ?? [] };
+    }),
+  );
+
+  const saturatedChunks: string[][] = [];
+  for (const { chunk, entries } of batchResults) {
+    collectFacts(entries, allResults);
+    if (entries.length >= PAGE_SIZE) {
+      saturatedChunks.push(chunk);
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Phase 3: Cursor-based pagination for saturated batches (power users).
+  // -----------------------------------------------------------------------
+  for (const chunk of saturatedChunks) {
+    if (allResults.size >= maxCandidates) break;
+    await paginateChunk(subgraphUrl, chunk, owner, allResults, maxCandidates, authKeyHex);
+  }
+
+  return Array.from(allResults.values());
+}
+
+/**
+ * Get fact count from the subgraph for dynamic pool sizing.
+ * Uses the globalStates entity for a lightweight single-row lookup
+ * instead of fetching and counting individual fact IDs.
+ */
+export async function getSubgraphFactCount(owner: string, authKeyHex?: string): Promise<number> {
+  const config = getSubgraphConfig();
+  const subgraphUrl = `${config.relayUrl}/v1/subgraph`;
+
+  const query = `
+    query FactCount {
+      globalStates(first: 1) {
+        totalFacts
+      }
+    }
+  `;
+
+  const data = await gqlQuery<{ globalStates?: Array<{ totalFacts: string }> }>(
+    subgraphUrl,
+    query,
+    {},
+    authKeyHex,
+  );
+
+  if (data?.globalStates && data.globalStates.length > 0) {
+    const count = parseInt(data.globalStates[0].totalFacts, 10);
+    return isNaN(count) ? 0 : count;
+  }
+
+  return 0;
+}

package/subgraph-store.ts

@@ -0,0 +1,346 @@
+/**
+ * Subgraph store path — writes facts on-chain via ERC-4337 UserOps.
+ *
+ * Used when the managed service is active (TOTALRECLAW_SELF_HOSTED is not
+ * "true"). Replaces the HTTP POST to /v1/store with an on-chain transaction
+ * flow.
+ *
+ * Builds UserOps client-side using `permissionless` + `viem` and submits
+ * them through the TotalReclaw relay server, which proxies bundler/paymaster
+ * JSON-RPC to Pimlico with its own API key. Clients never need a Pimlico key.
+ */
+
+import { createPublicClient, http, type Hex, type Address, type Chain } from 'viem';
+import { entryPoint07Address } from 'viem/account-abstraction';
+import { mnemonicToAccount } from 'viem/accounts';
+import { gnosis, gnosisChiado } from 'viem/chains';
+import { createSmartAccountClient } from 'permissionless';
+import { toSimpleSmartAccount } from 'permissionless/accounts';
+import { createPimlicoClient } from 'permissionless/clients/pimlico';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Default EventfulDataEdge contract address on Gnosis mainnet */
+const DEFAULT_DATA_EDGE_ADDRESS = '0xC445af1D4EB9fce4e1E61fE96ea7B8feBF03c5ca';
+
+/** Well-known ERC-4337 EntryPoint v0.7 address (same on all chains) */
+const DEFAULT_ENTRYPOINT_ADDRESS = '0x0000000071727De22E5E9d8BAf0edAc6f37da032';
+
+export interface SubgraphStoreConfig {
+  relayUrl: string;           // TotalReclaw relay server URL (proxies bundler + subgraph)
+  mnemonic: string;           // BIP-39 mnemonic for key derivation
+  cachePath: string;          // Hot cache file path
+  chainId: number;            // 100 for Gnosis mainnet, 10200 for Chiado testnet
+  dataEdgeAddress: string;    // EventfulDataEdge contract address
+  entryPointAddress: string;  // ERC-4337 EntryPoint v0.7
+  authKeyHex?: string;        // HKDF auth key for relay server Authorization header
+  rpcUrl?: string;            // Override chain RPC URL for public client reads
+  walletAddress?: string;     // Smart Account address for billing (X-Wallet-Address header)
+}
+
+export interface FactPayload {
+  id: string;
+  timestamp: string;
+  owner: string;           // Smart Account address (hex)
+  encryptedBlob: string;   // Hex-encoded AES-256-GCM ciphertext
+  blindIndices: string[];   // SHA-256 hashes (word + LSH)
+  decayScore: number;
+  source: string;
+  contentFp: string;
+  agentId: string;
+  encryptedEmbedding?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Protobuf encoding (unchanged)
+// ---------------------------------------------------------------------------
+
+/**
+ * Encode a fact payload as a minimal Protobuf wire format.
+ *
+ * Field numbers match server/proto/totalreclaw.proto:
+ *   1: id (string), 2: timestamp (string), 3: owner (string),
+ *   4: encrypted_blob (bytes), 5: blind_indices (repeated string),
+ *   6: decay_score (double), 7: is_active (bool), 8: version (int32),
+ *   9: source (string), 10: content_fp (string), 11: agent_id (string),
+ *   12: sequence_id (int64), 13: encrypted_embedding (string)
+ */
+export function encodeFactProtobuf(fact: FactPayload): Buffer {
+  const parts: Buffer[] = [];
+
+  // Helper: encode a string field
+  const writeString = (fieldNumber: number, value: string) => {
+    if (!value) return;
+    const data = Buffer.from(value, 'utf-8');
+    const key = (fieldNumber << 3) | 2; // wire type 2 = length-delimited
+    parts.push(encodeVarint(key));
+    parts.push(encodeVarint(data.length));
+    parts.push(data);
+  };
+
+  // Helper: encode a bytes field
+  const writeBytes = (fieldNumber: number, value: Buffer) => {
+    const key = (fieldNumber << 3) | 2;
+    parts.push(encodeVarint(key));
+    parts.push(encodeVarint(value.length));
+    parts.push(value);
+  };
+
+  // Helper: encode a double field (wire type 1 = 64-bit)
+  const writeDouble = (fieldNumber: number, value: number) => {
+    const key = (fieldNumber << 3) | 1;
+    parts.push(encodeVarint(key));
+    const buf = Buffer.alloc(8);
+    buf.writeDoubleLE(value);
+    parts.push(buf);
+  };
+
+  // Helper: encode a varint field (wire type 0)
+  const writeVarintField = (fieldNumber: number, value: number) => {
+    const key = (fieldNumber << 3) | 0;
+    parts.push(encodeVarint(key));
+    parts.push(encodeVarint(value));
+  };
+
+  // Encode fields
+  writeString(1, fact.id);
+  writeString(2, fact.timestamp);
+  writeString(3, fact.owner);
+  writeBytes(4, Buffer.from(fact.encryptedBlob, 'hex'));
+
+  for (const index of fact.blindIndices) {
+    writeString(5, index);
+  }
+
+  writeDouble(6, fact.decayScore);
+  writeVarintField(7, 1); // is_active = true
+  writeVarintField(8, 2); // version = 2
+  writeString(9, fact.source);
+  writeString(10, fact.contentFp);
+  writeString(11, fact.agentId);
+  // Field 12 (sequence_id) is assigned by the subgraph mapping, not the client
+  if (fact.encryptedEmbedding) {
+    writeString(13, fact.encryptedEmbedding);
+  }
+
+  return Buffer.concat(parts);
+}
+
+/** Encode an integer as a Protobuf varint */
+export function encodeVarint(value: number): Buffer {
+  const bytes: number[] = [];
+  let v = value >>> 0; // unsigned
+  while (v > 0x7f) {
+    bytes.push((v & 0x7f) | 0x80);
+    v >>>= 7;
+  }
+  bytes.push(v & 0x7f);
+  return Buffer.from(bytes);
+}
+
+// ---------------------------------------------------------------------------
+// Chain helpers
+// ---------------------------------------------------------------------------
+
+/** Resolve a viem Chain object from chain ID */
+function getChainFromId(chainId: number): Chain {
+  switch (chainId) {
+    case 100:
+      return gnosis;
+    case 10200:
+      return gnosisChiado;
+    default:
+      return gnosisChiado;
+  }
+}
+
+/** Build the relay bundler RPC URL from the relay server URL */
+function getRelayBundlerUrl(relayUrl: string): string {
+  return `${relayUrl}/v1/bundler`;
+}
+
+// ---------------------------------------------------------------------------
+// On-chain submission (Pimlico UserOps)
+// ---------------------------------------------------------------------------
+
+/**
+ * Submit a fact on-chain via ERC-4337 UserOp through the relay server.
+ *
+ * Builds a UserOp client-side using `permissionless` + `viem`:
+ * 1. Derives private key from mnemonic (BIP-39 + BIP-44 m/44'/60'/0'/0/0)
+ * 2. Creates a SimpleSmartAccount
+ * 3. Gets paymaster sponsorship (via relay proxy to Pimlico)
+ * 4. Signs and submits the UserOp to relay bundler endpoint
+ * 5. Waits for the transaction receipt
+ *
+ * The relay server proxies all bundler/paymaster JSON-RPC to Pimlico
+ * with its own API key. Clients never need a Pimlico API key.
+ */
+export async function submitFactOnChain(
+  protobufPayload: Buffer,
+  config: SubgraphStoreConfig,
+): Promise<{ txHash: string; userOpHash: string; success: boolean }> {
+  if (!config.relayUrl) {
+    throw new Error('Relay URL (TOTALRECLAW_SERVER_URL) is required for on-chain submission');
+  }
+
+  if (!config.mnemonic) {
+    throw new Error('Mnemonic (TOTALRECLAW_MASTER_PASSWORD) is required for on-chain submission');
+  }
+
+  const chain = getChainFromId(config.chainId);
+  const bundlerRpcUrl = getRelayBundlerUrl(config.relayUrl);
+  const dataEdgeAddress = config.dataEdgeAddress as Address;
+  const entryPointAddr = (config.entryPointAddress || entryPoint07Address) as Address;
+
+  // Build authenticated transport for relay server proxy
+  const headers: Record<string, string> = {
+    'X-TotalReclaw-Client': 'openclaw-plugin',
+  };
+  if (config.authKeyHex) headers['Authorization'] = `Bearer ${config.authKeyHex}`;
+  if (config.walletAddress) headers['X-Wallet-Address'] = config.walletAddress;
+
+  const authTransport = Object.keys(headers).length > 0
+    ? http(bundlerRpcUrl, { fetchOptions: { headers } })
+    : http(bundlerRpcUrl);
+
+  // 1. Derive EOA signer from mnemonic (BIP-44 m/44'/60'/0'/0/0)
+  const ownerAccount = mnemonicToAccount(config.mnemonic);
+
+  // 2. Create a public client for chain reads (using explicit RPC if configured,
+  //    NOT the bundler proxy which only supports ERC-4337 JSON-RPC methods)
+  const publicClient = createPublicClient({
+    chain,
+    transport: config.rpcUrl ? http(config.rpcUrl) : http(),
+  });
+
+  // 3. Create Pimlico client for bundler + paymaster operations (via relay)
+  const pimlicoClient = createPimlicoClient({
+    chain,
+    transport: authTransport,
+    entryPoint: {
+      address: entryPointAddr,
+      version: '0.7',
+    },
+  });
+
+  // 4. Create a SimpleSmartAccount (auto-generates initCode if undeployed)
+  const smartAccount = await toSimpleSmartAccount({
+    client: publicClient,
+    owner: ownerAccount,
+    entryPoint: {
+      address: entryPointAddr,
+      version: '0.7',
+    },
+  });
+
+  // 5. Create smart account client wired to relay bundler + paymaster
+  const smartAccountClient = createSmartAccountClient({
+    account: smartAccount,
+    chain,
+    bundlerTransport: authTransport,
+    // Paymaster sponsorship proxied through relay to Pimlico
+    paymaster: pimlicoClient,
+    userOperation: {
+      estimateFeesPerGas: async () => {
+        return (await pimlicoClient.getUserOperationGasPrice()).fast;
+      },
+    },
+  });
+
+  // 6. Send the transaction: Smart Account execute(dataEdgeAddress, 0, protobufPayload)
+  //    The DataEdge contract has a fallback() that emits Log(bytes), so the calldata
+  //    IS the protobuf payload directly (no function selector needed).
+  //    permissionless encodes the execute() call internally from to/value/data.
+  const calldata = `0x${protobufPayload.toString('hex')}` as Hex;
+
+  // Use sendUserOperation to get the userOpHash, then wait for receipt
+  const userOpHash = await smartAccountClient.sendUserOperation({
+    calls: [
+      {
+        to: dataEdgeAddress,
+        value: 0n,
+        data: calldata,
+      },
+    ],
+  });
+
+  // 7. Wait for the UserOp to be included in a transaction
+  const receipt = await pimlicoClient.waitForUserOperationReceipt({
+    hash: userOpHash,
+  });
+
+  return {
+    txHash: receipt.receipt.transactionHash,
+    userOpHash,
+    success: receipt.success,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if subgraph mode is enabled (i.e. using the managed service).
+ *
+ * Returns true when TOTALRECLAW_SELF_HOSTED is NOT set to "true".
+ * The managed service (subgraph mode) is the default.
+ */
+export function isSubgraphMode(): boolean {
+  return process.env.TOTALRECLAW_SELF_HOSTED !== 'true';
+}
+
+/**
+ * Get subgraph configuration from environment variables.
+ *
+ * After the relay refactor, clients only need:
+ *   - TOTALRECLAW_MASTER_PASSWORD -- BIP-39 mnemonic
+ *   - TOTALRECLAW_SERVER_URL -- relay server URL (default: https://api.totalreclaw.xyz)
+ *   - TOTALRECLAW_SELF_HOSTED -- set "true" to use self-hosted server (default: managed service)
+ *   - TOTALRECLAW_CHAIN_ID -- optional, defaults to 100 (Gnosis mainnet)
+ *
+ * Removed from client-side config (now server-side only):
+ *   - PIMLICO_API_KEY
+ *   - TOTALRECLAW_SUBGRAPH_ENDPOINT
+ */
+/**
+ * Derive the Smart Account address from a BIP-39 mnemonic.
+ * This is the on-chain owner identity used in the subgraph.
+ */
+export async function deriveSmartAccountAddress(mnemonic: string, chainId?: number): Promise<string> {
+  const chain: Chain = (chainId ?? 100) === 100 ? gnosis : gnosisChiado;
+  const ownerAccount = mnemonicToAccount(mnemonic);
+  const entryPointAddr = (process.env.TOTALRECLAW_ENTRYPOINT_ADDRESS || DEFAULT_ENTRYPOINT_ADDRESS) as Address;
+  const rpcUrl = process.env.TOTALRECLAW_RPC_URL;
+
+  const publicClient = createPublicClient({
+    chain,
+    transport: rpcUrl ? http(rpcUrl) : http(),
+  });
+
+  const smartAccount = await toSimpleSmartAccount({
+    client: publicClient,
+    owner: ownerAccount,
+    entryPoint: {
+      address: entryPointAddr,
+      version: '0.7',
+    },
+  });
+
+  return smartAccount.address.toLowerCase();
+}
+
+export function getSubgraphConfig(): SubgraphStoreConfig {
+  return {
+    relayUrl: process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz',
+    mnemonic: process.env.TOTALRECLAW_MASTER_PASSWORD || '',
+    cachePath: process.env.TOTALRECLAW_CACHE_PATH || `${process.env.HOME}/.totalreclaw/cache.enc`,
+    chainId: parseInt(process.env.TOTALRECLAW_CHAIN_ID || '100'),
+    dataEdgeAddress: process.env.TOTALRECLAW_DATA_EDGE_ADDRESS || DEFAULT_DATA_EDGE_ADDRESS,
+    entryPointAddress: process.env.TOTALRECLAW_ENTRYPOINT_ADDRESS || DEFAULT_ENTRYPOINT_ADDRESS,
+    rpcUrl: process.env.TOTALRECLAW_RPC_URL || undefined,
+  };
+}