@mseep/anything-analyzer 3.6.50

package/src/main/ai/crypto-script-extractor.ts

@@ -0,0 +1,206 @@
+import type { CapturedRequest, JsHookRecord, CryptoScriptSnippet } from '@shared/types'
+import type { RequestsRepo, JsHooksRepo } from '../db/repositories'
+
+const CONTEXT_LINES = 30
+
+const TIER1_PATTERNS = [
+  'crypto.subtle', 'CryptoJS', 'JSEncrypt', 'forge.cipher', 'forge.pki',
+  'forge.md', 'forge.hmac', 'new RSAKey', 'CryptoKey', 'sm2.doEncrypt',
+  'sm3(', 'sm4.encrypt',
+]
+
+const TIER2_PATTERNS = [
+  'encrypt(', 'decrypt(', '.sign(', '.verify(', '.digest(', 'hmac',
+  'AES', 'RSA', 'DES', 'SHA256', 'SHA1', 'SHA512', 'MD5', 'PBKDF2',
+  'createCipher', 'createDecipher', 'createHash', 'createHmac',
+  'publicKey', 'privateKey', 'secretKey',
+]
+
+const TIER3_PATTERNS = [
+  'btoa(', 'atob(', 'Base64', '.charCodeAt', 'fromCharCode',
+  'TextEncoder', 'encodeURIComponent',
+]
+
+const DEFAULT_BUDGET_CHARS = 20000
+
+/**
+ * CryptoScriptExtractor — Scans stored JS response bodies for crypto-related code
+ * and extracts relevant snippets with surrounding context.
+ */
+export class CryptoScriptExtractor {
+  constructor(
+    private requestsRepo: RequestsRepo,
+    private jsHooksRepo: JsHooksRepo,
+  ) {}
+
+  extract(sessionId: string, budgetChars: number = DEFAULT_BUDGET_CHARS): CryptoScriptSnippet[] {
+    const allRequests = this.requestsRepo.findBySession(sessionId)
+    const jsRequests = allRequests.filter(r => this.isJsRequest(r))
+
+    if (jsRequests.length === 0) return []
+
+    // Get hook call stacks for correlation
+    const hooks = this.jsHooksRepo.findBySession(sessionId)
+    const cryptoHooks = hooks.filter(h => h.hook_type === 'crypto' || h.hook_type === 'crypto_lib')
+    const hookScriptUrls = this.extractScriptUrlsFromHooks(cryptoHooks)
+
+    // Scan each JS file for crypto patterns
+    const allSnippets: (CryptoScriptSnippet & { hookCorrelation: number })[] = []
+
+    for (const req of jsRequests) {
+      if (!req.response_body) continue
+      const lines = req.response_body.split('\n')
+      const matches = this.findPatternMatches(lines)
+
+      if (matches.length === 0) continue
+
+      // Merge overlapping ranges
+      const merged = this.mergeRanges(matches, lines.length)
+
+      // Calculate hook correlation score
+      const scriptUrl = req.url
+      const hookCorrelation = hookScriptUrls.filter(u => scriptUrl.includes(u)).length
+
+      for (const range of merged) {
+        const content = lines.slice(range.start, range.end + 1).join('\n')
+        allSnippets.push({
+          scriptUrl,
+          lineRange: [range.start + 1, range.end + 1], // 1-based
+          content,
+          matchedPatterns: range.patterns,
+          tier: range.bestTier,
+          hookCorrelation,
+        })
+      }
+    }
+
+    // Sort: tier ASC, hookCorrelation DESC, then by content length ASC
+    allSnippets.sort((a, b) => {
+      if (a.tier !== b.tier) return a.tier - b.tier
+      if (a.hookCorrelation !== b.hookCorrelation) return b.hookCorrelation - a.hookCorrelation
+      return a.content.length - b.content.length
+    })
+
+    // Fill budget
+    const result: CryptoScriptSnippet[] = []
+    let usedChars = 0
+
+    for (const snippet of allSnippets) {
+      if (usedChars + snippet.content.length > budgetChars) {
+        // Try to fit a truncated version if at least 500 chars remain in budget
+        const remaining = budgetChars - usedChars
+        if (remaining >= 500) {
+          result.push({
+            scriptUrl: snippet.scriptUrl,
+            lineRange: snippet.lineRange,
+            content: snippet.content.substring(0, remaining) + '\n[TRUNCATED]',
+            matchedPatterns: snippet.matchedPatterns,
+            tier: snippet.tier,
+          })
+          usedChars = budgetChars
+        }
+        break
+      }
+      result.push({
+        scriptUrl: snippet.scriptUrl,
+        lineRange: snippet.lineRange,
+        content: snippet.content,
+        matchedPatterns: snippet.matchedPatterns,
+        tier: snippet.tier,
+      })
+      usedChars += snippet.content.length
+    }
+
+    return result
+  }
+
+  private isJsRequest(r: CapturedRequest): boolean {
+    if (r.method !== 'GET') return false
+    if (!r.response_body) return false
+    return /\.js(\?|$)/i.test(r.url) || (r.content_type?.includes('javascript') ?? false)
+  }
+
+  private findPatternMatches(lines: string[]): { line: number; tier: 1 | 2 | 3; pattern: string }[] {
+    const matches: { line: number; tier: 1 | 2 | 3; pattern: string }[] = []
+
+    for (let i = 0; i < lines.length; i++) {
+      const l = lines[i]
+      for (const p of TIER1_PATTERNS) {
+        if (l.includes(p)) {
+          matches.push({ line: i, tier: 1, pattern: p })
+          break // one match per line per tier is enough
+        }
+      }
+      for (const p of TIER2_PATTERNS) {
+        if (l.includes(p)) {
+          matches.push({ line: i, tier: 2, pattern: p })
+          break
+        }
+      }
+      for (const p of TIER3_PATTERNS) {
+        if (l.includes(p)) {
+          matches.push({ line: i, tier: 3, pattern: p })
+          break
+        }
+      }
+    }
+
+    return matches
+  }
+
+  private mergeRanges(
+    matches: { line: number; tier: 1 | 2 | 3; pattern: string }[],
+    totalLines: number,
+  ): { start: number; end: number; patterns: string[]; bestTier: 1 | 2 | 3 }[] {
+    if (matches.length === 0) return []
+
+    // Expand each match to a range with context
+    const ranges = matches.map(m => ({
+      start: Math.max(0, m.line - CONTEXT_LINES),
+      end: Math.min(totalLines - 1, m.line + CONTEXT_LINES),
+      patterns: [m.pattern],
+      bestTier: m.tier,
+    }))
+
+    // Sort by start position
+    ranges.sort((a, b) => a.start - b.start)
+
+    // Merge overlapping
+    const merged: typeof ranges = [ranges[0]]
+    for (let i = 1; i < ranges.length; i++) {
+      const prev = merged[merged.length - 1]
+      const curr = ranges[i]
+      if (curr.start <= prev.end + 1) {
+        // Overlapping, merge
+        prev.end = Math.max(prev.end, curr.end)
+        prev.patterns = [...new Set([...prev.patterns, ...curr.patterns])]
+        prev.bestTier = Math.min(prev.bestTier, curr.bestTier) as 1 | 2 | 3
+      } else {
+        merged.push(curr)
+      }
+    }
+
+    return merged
+  }
+
+  private extractScriptUrlsFromHooks(hooks: JsHookRecord[]): string[] {
+    const urls: string[] = []
+    for (const hook of hooks) {
+      if (!hook.call_stack) continue
+      // Parse stack frames like "at funcName (https://example.com/js/api.js:142:15)"
+      const urlMatches = hook.call_stack.match(/https?:\/\/[^\s:)]+\.js/g)
+      if (urlMatches) {
+        for (const url of urlMatches) {
+          // Extract just the path portion for matching
+          try {
+            const pathname = new URL(url).pathname
+            if (!urls.includes(pathname)) urls.push(pathname)
+          } catch {
+            if (!urls.includes(url)) urls.push(url)
+          }
+        }
+      }
+    }
+    return urls
+  }
+}

package/src/main/ai/data-assembler.ts

@@ -0,0 +1,205 @@
+import type { CapturedRequest, JsHookRecord, StorageSnapshot, AssembledData, FilteredRequest, StorageDiff, AuthChainItem, RequestSummary } from '@shared/types'
+import type { RequestsRepo, JsHooksRepo, StorageSnapshotsRepo } from '../db/repositories'
+import { SceneDetector } from './scene-detector'
+import { CryptoScriptExtractor } from './crypto-script-extractor'
+
+const STATIC_EXTENSIONS = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot|ico|map)(\?|$)/i
+const API_CONTENT_TYPES = ['json', 'form-urlencoded', 'multipart']
+const TOKEN_BUDGET = 30000
+const CHARS_PER_TOKEN = 4
+const CRYPTO_BUDGET_CHARS = 20000
+
+/**
+ * DataAssembler — Reads session data from SQLite, filters, associates, and budgets it.
+ */
+export class DataAssembler {
+  constructor(private requestsRepo: RequestsRepo, private jsHooksRepo: JsHooksRepo, private storageSnapshotsRepo: StorageSnapshotsRepo) {}
+
+  assemble(sessionId: string): AssembledData {
+    const allRequests = this.requestsRepo.findBySession(sessionId)
+    const allHooks = this.jsHooksRepo.findBySession(sessionId)
+    const allSnapshots = this.storageSnapshotsRepo.findBySession(sessionId)
+
+    const filteredRequests = allRequests.filter(r => this.isRelevantRequest(r))
+
+    const assembledRequests = filteredRequests.map(r => {
+      const relatedHooks = allHooks.filter(h => Math.abs(h.timestamp - r.timestamp) <= 2000)
+      return {
+        seq: r.sequence, method: r.method, url: r.url,
+        headers: this.safeParseJson(r.request_headers) || {},
+        body: r.request_body, status: r.status_code,
+        responseHeaders: r.response_headers ? this.safeParseJson(r.response_headers) : null,
+        responseBody: r.response_body, hooks: relatedHooks
+      }
+    })
+
+    const storageDiff = {
+      cookies: this.calcDiff(allSnapshots, 'cookie'),
+      localStorage: this.calcDiff(allSnapshots, 'localStorage'),
+      sessionStorage: this.calcDiff(allSnapshots, 'sessionStorage')
+    }
+
+    this.applyTokenBudget(assembledRequests)
+
+    // Extract crypto-related JS snippets from stored JS response bodies
+    const cryptoExtractor = new CryptoScriptExtractor(this.requestsRepo, this.jsHooksRepo)
+    const cryptoScripts = cryptoExtractor.extract(sessionId, CRYPTO_BUDGET_CHARS)
+
+    const estimatedTokens = this.estimateTokens(assembledRequests, storageDiff)
+
+    return {
+      requests: assembledRequests,
+      storageDiff,
+      estimatedTokens,
+      sceneHints: new SceneDetector().detect(assembledRequests),
+      streamingRequests: (() => {
+        const rawRequestMap = new Map(allRequests.map(r => [r.sequence, r]))
+        return assembledRequests.filter(r => {
+          const rawReq = rawRequestMap.get(r.seq)
+          return rawReq?.is_streaming || rawReq?.is_websocket
+        })
+      })(),
+      authChain: this.extractAuthChain(assembledRequests),
+      cryptoScripts,
+    }
+  }
+
+  private isRelevantRequest(r: CapturedRequest): boolean {
+    if (r.method !== 'GET') return true
+    if (STATIC_EXTENSIONS.test(r.url)) return false
+    if (r.content_type && API_CONTENT_TYPES.some(t => r.content_type!.includes(t))) return true
+    if (r.content_type?.includes('html')) return true
+    if (r.request_body) return true
+    return false
+  }
+
+  private calcDiff(snapshots: StorageSnapshot[], type: string): StorageDiff {
+    const typed = snapshots.filter(s => s.storage_type === type)
+    if (typed.length < 2) return { added: {}, changed: {}, removed: [] }
+    const first = this.safeParseJson(typed[0].data) || {}
+    const last = this.safeParseJson(typed[typed.length - 1].data) || {}
+    const added: Record<string, string> = {}, changed: Record<string, { old: string; new: string }> = {}, removed: string[] = []
+    for (const key of Object.keys(last)) {
+      if (!(key in first)) added[key] = String(last[key])
+      else if (JSON.stringify(first[key]) !== JSON.stringify(last[key])) changed[key] = { old: String(first[key]), new: String(last[key]) }
+    }
+    for (const key of Object.keys(first)) { if (!(key in last)) removed.push(key) }
+    return { added, changed, removed }
+  }
+
+  private applyTokenBudget(requests: FilteredRequest[]): void {
+    let totalChars = requests.reduce((sum, r) => sum + JSON.stringify(r.headers).length + r.url.length + (r.body?.length || 0) + (r.responseBody?.length || 0), 0)
+    const budgetChars = TOKEN_BUDGET * CHARS_PER_TOKEN
+    if (totalChars <= budgetChars) return
+    const sorted = [...requests].filter(r => r.responseBody && r.responseBody.length > 500).sort((a, b) => (b.responseBody?.length || 0) - (a.responseBody?.length || 0))
+    for (const req of sorted) {
+      if (totalChars <= budgetChars) break
+      const currentLen = req.responseBody?.length || 0
+      const truncLen = Math.max(500, Math.floor(currentLen / 4))
+      req.responseBody = req.responseBody!.substring(0, truncLen) + '\n[TRUNCATED FOR TOKEN BUDGET]'
+      totalChars -= (currentLen - truncLen)
+    }
+  }
+
+  private estimateTokens(requests: FilteredRequest[], storageDiff: AssembledData['storageDiff']): number {
+    let chars = requests.reduce((sum, r) => sum + JSON.stringify(r).length, 0) + JSON.stringify(storageDiff).length
+    return Math.ceil(chars / CHARS_PER_TOKEN)
+  }
+
+  private safeParseJson(json: string | null): Record<string, any> | null {
+    if (!json) return null
+    try { return JSON.parse(json) } catch { return null }
+  }
+
+  private extractAuthChain(requests: FilteredRequest[]): AuthChainItem[] {
+    const authChain: AuthChainItem[] = []
+
+    for (const req of requests) {
+      // 检查响应中是否返回了 token
+      if (req.responseBody) {
+        try {
+          const data = JSON.parse(req.responseBody)
+          if (data.access_token) {
+            authChain.push({
+              source: `${req.method} ${new URL(req.url).pathname} 响应`,
+              credentialType: 'Bearer Token',
+              credential: this.maskCredential(data.access_token),
+              consumers: []
+            })
+          }
+          if (data.refresh_token) {
+            authChain.push({
+              source: `${req.method} ${new URL(req.url).pathname} 响应`,
+              credentialType: 'Refresh Token',
+              credential: this.maskCredential(data.refresh_token),
+              consumers: []
+            })
+          }
+        } catch { /* non-JSON response */ }
+      }
+
+      // 检查响应中的 Set-Cookie
+      if (req.responseHeaders) {
+        const rawSetCookie = req.responseHeaders['set-cookie'] || req.responseHeaders['Set-Cookie']
+        if (rawSetCookie) {
+          const cookies = Array.isArray(rawSetCookie) ? rawSetCookie : [rawSetCookie]
+          for (const cookie of cookies) {
+            const cookieName = String(cookie).split('=')[0]
+            authChain.push({
+              source: `${req.method} ${new URL(req.url).pathname} Set-Cookie`,
+              credentialType: 'Session Cookie',
+              credential: `${cookieName}=...`,
+              consumers: []
+            })
+          }
+        }
+      }
+    }
+
+    // 标记 consumers：哪些后续请求使用了这些凭据
+    for (const req of requests) {
+      const authHeader = req.headers['authorization'] || req.headers['Authorization'] || ''
+      if (authHeader.startsWith('Bearer ')) {
+        const token = authHeader.substring(7)
+        const matchingItem = authChain.find(a => a.credentialType === 'Bearer Token' && token.startsWith(a.credential.substring(0, 8)))
+        if (matchingItem) {
+          try { matchingItem.consumers.push(new URL(req.url).pathname) } catch { matchingItem.consumers.push(req.url) }
+        }
+      }
+    }
+
+    return authChain
+  }
+
+  private maskCredential(value: string): string {
+    if (value.length <= 16) return '***'
+    return value.substring(0, 8) + '...' + value.substring(value.length - 8)
+  }
+
+  /**
+   * 从已组装数据中提取轻量请求摘要（用于 Phase 1 预过滤）
+   */
+  extractSummaries(data: AssembledData): RequestSummary[] {
+    return data.requests.map(r => ({
+      seq: r.seq,
+      method: r.method,
+      url: r.url,
+      status: r.status,
+      contentType: r.responseHeaders?.['content-type'] ?? null,
+    }))
+  }
+
+  /**
+   * 按序号过滤已组装数据，保留全局上下文（sceneHints、authChain、storageDiff、cryptoScripts）
+   */
+  filterBySeqs(data: AssembledData, selectedSeqs: number[]): AssembledData {
+    const seqSet = new Set(selectedSeqs)
+    const filteredRequests = data.requests.filter(r => seqSet.has(r.seq))
+    return {
+      ...data,
+      requests: filteredRequests,
+      streamingRequests: data.streamingRequests.filter(r => seqSet.has(r.seq)),
+      estimatedTokens: this.estimateTokens(filteredRequests, data.storageDiff),
+    }
+  }
+}