mcp-scraper 0.97.1 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -2434
- package/README.md +16 -412
- package/dist/bin/api-server.js +3 -2
- package/dist/bin/mcp-scraper-cli.js +33 -27
- package/dist/bin/mcp-scraper-core.js +9 -1
- package/dist/bin/mcp-scraper-install.js +17 -2
- package/dist/bin/mcp-stdio-server.js +9 -1
- package/dist/bin/paa-harvest.js +8 -1
- package/dist/index.cjs +5 -21830
- package/dist/index.d.ts +14 -607
- package/dist/index.js +5 -39
- package/dist/transport.d.ts +35 -0
- package/package.json +11 -51
- package/THIRD_PARTY_NOTICES.html +0 -203
- package/dist/actions-5X5CQAKO.js +0 -1
- package/dist/artifacts-4ZDNP2JW.js +0 -1
- package/dist/artifacts-N62OPIT5.js +0 -1
- package/dist/artifacts-PSADD5OC.js +0 -1
- package/dist/artifacts-RUR3OHAK.js +0 -1
- package/dist/blob-store-GIH26EDO.js +0 -1
- package/dist/bundle-GS657HXG.js +0 -1
- package/dist/chunk-24YDQ3EO.js +0 -5
- package/dist/chunk-2L7CHN5N.js +0 -4
- package/dist/chunk-3F7AFM53.js +0 -1
- package/dist/chunk-4AEW53RU.js +0 -954
- package/dist/chunk-4HZXATHB.js +0 -27
- package/dist/chunk-4YZ74SJ6.js +0 -1
- package/dist/chunk-4ZVO6GX4.js +0 -1
- package/dist/chunk-55ZANTHL.js +0 -3
- package/dist/chunk-5RQWFYXY.js +0 -930
- package/dist/chunk-5TQINRZN.js +0 -12
- package/dist/chunk-5UQAL7HV.js +0 -41
- package/dist/chunk-6D25Q5BR.js +0 -21728
- package/dist/chunk-6N5EC7ZO.js +0 -1
- package/dist/chunk-7POHNTVW.js +0 -1
- package/dist/chunk-7SR7MEU5.js +0 -62
- package/dist/chunk-ANQ4JJVI.js +0 -71
- package/dist/chunk-B3JRL45T.js +0 -1
- package/dist/chunk-B4ZKPLSP.js +0 -1
- package/dist/chunk-B5Y26E3A.js +0 -1
- package/dist/chunk-C3JWX6KM.js +0 -10
- package/dist/chunk-C7MGJUTU.js +0 -1
- package/dist/chunk-CNSVZTTT.js +0 -13
- package/dist/chunk-EYGA6LOU.js +0 -1
- package/dist/chunk-F3M7INSN.js +0 -41
- package/dist/chunk-FLAWC5PF.js +0 -5
- package/dist/chunk-FM7BX2LU.js +0 -159
- package/dist/chunk-HCK6PLM3.js +0 -3
- package/dist/chunk-HNNYXMSI.js +0 -1
- package/dist/chunk-IFPION5D.js +0 -121
- package/dist/chunk-IPQOBOFY.js +0 -1
- package/dist/chunk-J7XDXIB6.js +0 -163
- package/dist/chunk-JHDD6R3O.js +0 -8
- package/dist/chunk-JMZIDHYQ.js +0 -2
- package/dist/chunk-K3VDGJNK.js +0 -1
- package/dist/chunk-KBZY2OGP.js +0 -11
- package/dist/chunk-KJQXUZ4Y.js +0 -4
- package/dist/chunk-KXJCGV3H.js +0 -1
- package/dist/chunk-NALXKTUU.js +0 -1
- package/dist/chunk-NNOX3FYG.js +0 -198
- package/dist/chunk-NQFTETVC.js +0 -189
- package/dist/chunk-OILOHQNL.js +0 -2
- package/dist/chunk-OLYPLX5C.js +0 -1
- package/dist/chunk-P6CDR3RM.js +0 -1
- package/dist/chunk-PKQHHCO4.js +0 -20
- package/dist/chunk-QCBCOEH5.js +0 -1
- package/dist/chunk-QER2ELZP.js +0 -1
- package/dist/chunk-RDHGQ2WZ.js +0 -1
- package/dist/chunk-SDQW7IJN.js +0 -1
- package/dist/chunk-SEZETSB3.js +0 -1
- package/dist/chunk-SJNSTO3C.js +0 -42
- package/dist/chunk-TZYITYL4.js +0 -2
- package/dist/chunk-UIOSADPB.js +0 -1718
- package/dist/chunk-UNEJYZ4T.js +0 -102
- package/dist/chunk-UUJYICAJ.js +0 -1
- package/dist/chunk-VVSHZV5S.js +0 -73
- package/dist/chunk-WNUXMPVX.js +0 -1
- package/dist/chunk-XNTAHMPA.js +0 -218
- package/dist/chunk-ZDECILF6.js +0 -16
- package/dist/chunk-ZNPGM3XE.js +0 -4
- package/dist/compatibility-UPPKZ4KY.js +0 -1
- package/dist/contracts-GUXGFCVV.js +0 -1
- package/dist/core-schema-Y47YFKH2.js +0 -899
- package/dist/direct-completion-ZVPPEBDU.js +0 -1
- package/dist/dispatch-GRMMPKQD.js +0 -1
- package/dist/editorial-reading-room/assets/app.js +0 -433
- package/dist/editorial-reading-room/assets/index.html +0 -139
- package/dist/editorial-reading-room/assets/styles.css +0 -1091
- package/dist/index.d.cts +0 -607
- package/dist/jobs-NRZHGQ7I.js +0 -1
- package/dist/media-artifacts-67CJWI7O.js +0 -1
- package/dist/metering-YHZ46GV4.js +0 -1
- package/dist/migrations-3AKEC5QE.js +0 -1
- package/dist/queue-2VKW7SGH.js +0 -1
- package/dist/reconciliation-QC6SILLR.js +0 -1
- package/dist/report-artifact-offload-PQNFDNPL.js +0 -1
- package/dist/repository-CYXVMYAI.js +0 -1
- package/dist/repository-UKDA4IWP.js +0 -1
- package/dist/server-EYMJROPX.js +0 -5544
- package/dist/settlement-47EUTSGX.js +0 -1
- package/dist/vendor-errors-647LSH6H.js +0 -1
- package/dist/worker-AWJF7UYR.js +0 -1
- package/public/.well-known/xray-install.json +0 -183
- package/public/agent-install.md +0 -200
package/dist/chunk-J7XDXIB6.js
DELETED
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
import{D as Oe,F as Re,G as Ce,H as j,c as R,h as Ee,i as C,j as y,k as ye,l as Se,m as P,n as he,o as Ae,p as we,q as T,r as xe,s as w,t as p,u as Ne,x as Ie,y as De,z as ke}from"./chunk-SJNSTO3C.js";import{L as ee}from"./chunk-6D25Q5BR.js";import{b as F}from"./chunk-C3JWX6KM.js";import{b as re,c as te}from"./chunk-QCBCOEH5.js";import{d as O}from"./chunk-KBZY2OGP.js";import{a as ue,b as ce,c as pe,f as me}from"./chunk-4YZ74SJ6.js";import{d as de}from"./chunk-JHDD6R3O.js";import{b as be,d as ge,e as D,f as ve,j as k,p as fe}from"./chunk-7SR7MEU5.js";import{d as oe,e as ne,f as ie,g as se,i as I,l as ae,s as le,t as b}from"./chunk-ANQ4JJVI.js";import{a as g,b as E,c}from"./chunk-FLAWC5PF.js";import{c as W,d as B}from"./chunk-B4ZKPLSP.js";import{a as V,b as X,c as Z}from"./chunk-VVSHZV5S.js";import{b as Y}from"./chunk-4ZVO6GX4.js";import{b as N,d as z,g as m,h as A,j as K,m as G,o as Q}from"./chunk-C7MGJUTU.js";import{a}from"./chunk-WNUXMPVX.js";import{b as _e}from"./chunk-B3JRL45T.js";import{c as h,d as f}from"./chunk-HNNYXMSI.js";import{randomUUID as tr}from"crypto";async function Ar(e,r,t){let n=tr();return await a().execute({sql:"INSERT INTO jobs (id, user_id, query, options, status, started_at) VALUES (?, ?, ?, ?, 'running', datetime('now'))",args:[n,e,r,JSON.stringify(t)]}),n}async function wr(e,r,t,n){let s=await a().execute({sql:`INSERT INTO jobs (id, user_id, query, options, status, started_at)
|
|
2
|
-
VALUES (?, ?, ?, ?, 'running', datetime('now'))
|
|
3
|
-
ON CONFLICT(id) DO NOTHING`,args:[e,r,t,JSON.stringify(n)]}),i=await c(e,r);if(!i)throw new Error(`getOrCreateRunningJob: row missing immediately after insert/lookup for ${e}`);return{job:i,created:s.rowsAffected===1}}async function xr(e,r,t){await a().execute({sql:`UPDATE jobs
|
|
4
|
-
SET status = 'running', started_at = COALESCE(started_at, datetime('now')),
|
|
5
|
-
options = CASE WHEN ? = 'runtime' THEN json_set(options, '$.executionOwner', 'runtime') ELSE options END
|
|
6
|
-
WHERE id = ?
|
|
7
|
-
AND (status = 'pending' OR (
|
|
8
|
-
status = 'running' AND ? = 'runtime'
|
|
9
|
-
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
10
|
-
AND EXISTS (SELECT 1 FROM operation_executions e JOIN operation_runs r ON r.id = e.run_id
|
|
11
|
-
WHERE e.run_id = 'extract-url-execution:' || jobs.id AND r.user_id = jobs.user_id
|
|
12
|
-
AND r.status = 'running' AND e.phase = 'claimed'
|
|
13
|
-
AND e.expires_at_ms <= CAST(strftime('%s', 'now') AS INTEGER) * 1000)
|
|
14
|
-
))
|
|
15
|
-
AND (json_extract(options, '$.executionOwner') = ?
|
|
16
|
-
OR (? = 'runtime' AND json_extract(options, '$.executionOwner') = 'inngest'))
|
|
17
|
-
AND json_extract(options, '$.jobKind') = ?`,args:[r,e,r,r,r,t]});let n=await c(e);if(!(!n||n.options.executionOwner!==r||n.options.jobKind!==t))return n}async function Nr(e=25){let r=Math.max(1,Math.min(100,Math.floor(e)));return(await a().execute({sql:`SELECT * FROM jobs
|
|
18
|
-
WHERE status IN ('pending', 'running')
|
|
19
|
-
AND json_extract(options, '$.executionOwner') IN ('runtime', 'inngest')
|
|
20
|
-
AND json_extract(options, '$.jobKind') = 'extract_url'
|
|
21
|
-
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
22
|
-
AND EXISTS (SELECT 1 FROM billing_debits d
|
|
23
|
-
WHERE d.idempotency_key = json_extract(jobs.options, '$.billingDebitKey')
|
|
24
|
-
AND d.user_id = jobs.user_id AND d.status = 'applied'
|
|
25
|
-
AND d.operation = ? AND d.amount_mc = ?)
|
|
26
|
-
AND (
|
|
27
|
-
NOT EXISTS (SELECT 1 FROM operation_executions e
|
|
28
|
-
WHERE e.run_id = 'extract-url-execution:' || jobs.id)
|
|
29
|
-
AND (status = 'pending' OR json_extract(options, '$.executionOwner') = 'runtime')
|
|
30
|
-
OR EXISTS (SELECT 1 FROM operation_executions e JOIN operation_runs r ON r.id = e.run_id
|
|
31
|
-
WHERE e.run_id = 'extract-url-execution:' || jobs.id AND r.user_id = jobs.user_id
|
|
32
|
-
AND r.status = 'running' AND e.phase = 'claimed'
|
|
33
|
-
AND e.expires_at_ms <= CAST(strftime('%s', 'now') AS INTEGER) * 1000)
|
|
34
|
-
)
|
|
35
|
-
AND created_at <= datetime('now', '-30 seconds')
|
|
36
|
-
AND (status = 'pending' OR datetime(COALESCE(started_at, created_at)) <= datetime('now', '-30 seconds'))
|
|
37
|
-
AND datetime(COALESCE(json_extract(options, '$.extractUrlLastDispatchAt'), '1970-01-01')) <= datetime('now', '-30 seconds')
|
|
38
|
-
ORDER BY created_at
|
|
39
|
-
LIMIT ?`,args:[ce.EXTRACT_URL,ue.page_scrape,r]})).rows.map(n=>E(g(n)))}async function Ir(e,r){return(await a().execute({sql:`UPDATE jobs
|
|
40
|
-
SET options = json_set(options,
|
|
41
|
-
'$.extractUrlDispatchAttempts', COALESCE(json_extract(options, '$.extractUrlDispatchAttempts'), 0) + 1,
|
|
42
|
-
'$.extractUrlLastDispatchAt', ?,
|
|
43
|
-
'$.extractUrlLastDispatchError', ?)
|
|
44
|
-
WHERE id = ? AND (status = 'pending' OR (status = 'running' AND (
|
|
45
|
-
json_extract(options, '$.executionOwner') = 'runtime'
|
|
46
|
-
OR EXISTS (SELECT 1 FROM operation_executions e JOIN operation_runs r ON r.id = e.run_id
|
|
47
|
-
WHERE e.run_id = 'extract-url-execution:' || jobs.id AND r.user_id = jobs.user_id
|
|
48
|
-
AND r.status = 'running' AND e.phase = 'claimed'
|
|
49
|
-
AND e.expires_at_ms <= CAST(strftime('%s', 'now') AS INTEGER) * 1000)
|
|
50
|
-
)))
|
|
51
|
-
AND json_extract(options, '$.executionOwner') IN ('runtime', 'inngest')
|
|
52
|
-
AND json_extract(options, '$.jobKind') = 'extract_url'`,args:[new Date().toISOString(),r??null,e]})).rowsAffected===1}async function Dr(e){return(await a().execute({sql:"SELECT * FROM jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[e]})).rows.map(t=>E(g(t)))}async function Pe(){let e=a(),r=await e.execute(`
|
|
53
|
-
SELECT * FROM jobs
|
|
54
|
-
WHERE status = 'pending'
|
|
55
|
-
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'runtime', 'inngest')
|
|
56
|
-
AND json_extract(options,'$.queueExecutionProtocol') IS NOT 0
|
|
57
|
-
ORDER BY created_at
|
|
58
|
-
LIMIT 1
|
|
59
|
-
`),t=r.rows[0]?g(r.rows[0]):void 0;return t?(await e.execute({sql:"UPDATE jobs SET status = 'running', started_at = datetime('now') WHERE id = ? AND status = 'pending'",args:[t.id]})).rowsAffected===0?void 0:{...t,status:"running"}:void 0}async function kr(e,r){await a().execute({sql:`UPDATE jobs SET status = 'done', result = ?, error = NULL, public_error_json = NULL, completed_at = datetime('now') WHERE id = ? AND ${O}`,args:[JSON.stringify(r),e]})}async function Or(e,r,t){await a().execute({sql:`UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ? AND ${O}`,args:[r,z(t),e]})}async function Tr(e=25){let r=Math.max(1,Math.min(100,Math.floor(e)));return(await a().execute({sql:`SELECT * FROM jobs
|
|
60
|
-
WHERE status IN ('done', 'failed', 'cancelled')
|
|
61
|
-
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
62
|
-
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
63
|
-
AND CAST(json_extract(options, '$.billingHoldMc') AS INTEGER) > 0
|
|
64
|
-
AND json_extract(options, '$.billingSettledAt') IS NULL
|
|
65
|
-
ORDER BY completed_at DESC
|
|
66
|
-
LIMIT ?`,args:[r]})).rows.map(n=>E(g(n)))}async function Te(e,r=25,t="1"){if(!/^[a-z_]+$/.test(e)||!t.trim()||t.length>8192)throw Error("Invalid intent selection");return(await a().execute({sql:`SELECT r.id,r.user_id,r.job_id FROM operation_runs r
|
|
67
|
-
JOIN operation_executions e ON e.run_id=r.id
|
|
68
|
-
WHERE r.tool=? AND r.status='running' AND e.phase='claimed'
|
|
69
|
-
AND e.expires_at_ms <= CAST(strftime('%s','now') AS INTEGER)*1000
|
|
70
|
-
AND json_extract(r.normalized_flags_json,'$._storeGateRequired') IS 1 AND (${t})
|
|
71
|
-
ORDER BY r.started_at LIMIT ?`,args:[e,Math.max(1,Math.min(100,Math.floor(r)))]})).rows.map(s=>({runId:String(s.id),userId:Number(s.user_id),jobId:String(s.job_id)}))}function je(e){let r=JSON.parse(f(e.options)),t=r.serpOnly===!0&&!r.serpIdentity;return{profile:"queue",jobId:e.jobId,userId:e.userId,tool:"search_serp",requestFingerprint:e.requestFingerprint,options:r,callbackUrl:e.callbackUrl??null,deadlineAt:0,budgetMs:W(Number(r.maxQuestions??30),!0,r.mode,r.pages,!!r.recency,t).serverMs,hold:{userId:e.userId,key:`serp-search:${e.jobId}:hold`,amountMc:e.amountMc,operation:"serp",description:`SERP search: ${r.query}`.slice(0,500)},refundOperation:"serp_refund",sourceSurface:"serp_search"}}var Gr=e=>j(je(e));async function or(e,r){let t;try{t=await b(`harvest:${r}`,e,"search_serp",r)}catch{return{kind:"unknown",reply:{status:503,body:{job_id:r,status:"pending",executionState:"unknown",hold_state:"unconfirmed",status_tool:"harvest_paa_status",...A({errorCode:"response_lost",retryable:!0})}}}}return t?.flags.syncExecutionProtocol!==1||t.flags.intent?.profile!=="queue"?null:j(t.flags.intent,!1)}async function Me(){for(let r of await Te("search_serp",1,`json_extract(r.normalized_flags_json,'$.intent.profile')='queue' AND (EXISTS(SELECT 1 FROM (SELECT r.user_id AS user_id,json_extract(r.normalized_flags_json,'$.intent') AS normalized_flags_json) saved WHERE ${de("saved.user_id","json_extract(saved.normalized_flags_json,'$.hold.key')")}))`))await c(r.jobId,r.userId)||await or(r.userId,r.jobId);let e=(await a().execute(`SELECT j.id,j.user_id FROM jobs j JOIN operation_runs r ON r.id='harvest:'||j.id AND r.user_id=j.user_id JOIN operation_executions x ON x.run_id=r.id
|
|
72
|
-
WHERE json_extract(j.options,'$.queueExecutionProtocol')=1 AND x.phase IN ('effects_started','unknown') AND x.expires_at_ms<=CAST(strftime('%s','now') AS INTEGER)*1000
|
|
73
|
-
ORDER BY j.created_at LIMIT 1`)).rows[0];return e?c(String(e.id),Number(e.user_id)):void 0}async function Le(e,r){if(e.options.queueExecutionProtocol!==1)return await a().execute({sql:"UPDATE jobs SET options=json_set(options,'$.queueExecutionProtocol',0,'$.executionState','unknown','$.holdState','unconfirmed') WHERE id=? AND user_id=? AND json_extract(options,'$.queueExecutionProtocol') IS NULL",args:[e.id,e.user_id]}),{completed:!1,replayed:!1,reply:null};let t=await Re(e);if(t)return{completed:(await c(e.id,e.user_id))?.options.executionState==="finalized"&&t.status===200,replayed:!0,reply:t};let n=e.options,s=Object.fromEntries(Object.entries(n).filter(([d])=>!["executionOwner","queueExecutionProtocol","billingHoldMc","billingDebitKey","requestFingerprint","compatDeadlineAt","executionState","holdState"].includes(d))),i=je({jobId:e.id,userId:e.user_id,options:s,requestFingerprint:String(n.requestFingerprint),amountMc:Number(n.billingHoldMc),callbackUrl:e.callback_url}),o=await Ce(i,async d=>{let u={value:null},_=n.serpOnly===!0&&!n.serpIdentity,v=`harvest:${e.id}`,x=null,J=AbortSignal.timeout(Math.max(0,d.deadlineAt-Date.now()));if(_&&!De({pages:n.pages===2?2:1,recency:n.recency==="week"||n.recency==="month"?n.recency:void 0,mode:n.mode,token:r.apifyToken}))throw Error("Search providers are not configured");let er=await r.withCostContext({op:"serp",userId:e.user_id,headlessSentOut:u,operationRunId:v,operationAccountScope:"production-default"},async()=>_?ke({query:String(n.query??e.query),pages:n.pages===2?2:1,mode:n.mode,recency:n.recency==="week"||n.recency==="month"?n.recency:void 0,token:r.apifyToken,operationRunId:v,accountScope:"production-default",signal:J,onAttemptEvent:r.onAttemptEvent,physicalFactory:d.physicalFactory}).then($=>(x=$,$.result)):ee({...n,physicalExecution:await d.physicalFactory(),\u006b\u0065\u0072\u006e\u0065\u006cApiKey:r.\u006b\u0065\u0072\u006e\u0065\u006cApiKey,headless:!0,format:"json",outputDir:"/tmp/paa-output-api",signal:J,onAttemptEvent:r.onAttemptEvent}));if(_&&!x)throw Error("Search provider billing receipt is missing");let U=x,rr=_?me(U.deliveredPages,U.deliveryProvider):pe(u.value);return{result:er,amountMc:Math.min(Number(n.billingHoldMc),rr)}},r.classifyFailure);return{completed:(await c(e.id,e.user_id))?.options.executionState==="finalized"&&!!(o.body.status&&["done","failed","cancelled"].includes(String(o.body.status))),replayed:!1,reply:o}}import{randomUUID as nr}from"crypto";function S(e){return e==null?null:Number(e)===1}function He(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function qe(e){return{id:String(e.id),job_id:String(e.job_id),user_id:Number(e.user_id),attempt_number:Number(e.attempt_number),max_attempts:Number(e.max_attempts),query:String(e.query),location:e.location!=null?String(e.location):null,max_questions:Number(e.max_questions),status:String(e.status),outcome:e.outcome!=null?String(e.outcome):null,\u006b\u0065\u0072\u006e\u0065\u006c_session_id:e.\u006b\u0065\u0072\u006e\u0065\u006c_session_id!=null?String(e.\u006b\u0065\u0072\u006e\u0065\u006c_session_id):null,question_count:e.question_count!=null?Number(e.question_count):null,duration_ms:e.duration_ms!=null?Number(e.duration_ms):null,error:e.error!=null?String(e.error):null,will_retry:S(e.will_retry),\u006b\u0065\u0072\u006e\u0065\u006c_delete_started:Number(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_started??0)===1,\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded:S(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded),\u006b\u0065\u0072\u006e\u0065\u006c_delete_error:e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_error!=null?String(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_error):null,browser_close_succeeded:S(e.browser_close_succeeded),browser_close_error:e.browser_close_error!=null?String(e.browser_close_error):null,browser_provider:e.browser_provider!=null?String(e.browser_provider):null,provider_session_id:e.provider_session_id!=null?String(e.provider_session_id):null,provider_session_observed_at:e.provider_session_observed_at!=null?String(e.provider_session_observed_at):null,provider_lookup_status:e.provider_lookup_status!=null?String(e.provider_lookup_status):null,provider_status:e.provider_status!=null?String(e.provider_status):null,provider_error_code:e.provider_error_code!=null?String(e.provider_error_code):null,provider_error_message:e.provider_error_message!=null?String(e.provider_error_message):null,provider_reason_source:e.provider_reason_source!=null?String(e.provider_reason_source):null,provider_duration_ms:e.provider_duration_ms!=null?Number(e.provider_duration_ms):null,provider_bandwidth_bytes:e.provider_bandwidth_bytes!=null?Number(e.provider_bandwidth_bytes):null,provider_captcha:S(e.provider_captcha),provider_checked_at:e.provider_checked_at!=null?String(e.provider_checked_at):null,provider_detail:He(e.provider_detail_json),provider_disconnect_observed:S(e.provider_disconnect_observed),provider_disconnect_message:e.provider_disconnect_message!=null?String(e.provider_disconnect_message):null,debug:He(e.debug_json),created_at:String(e.created_at),completed_at:e.completed_at!=null?String(e.completed_at):null}}async function Je(e){await a().execute({sql:`
|
|
74
|
-
INSERT INTO harvest_attempts (
|
|
75
|
-
id, job_id, user_id, attempt_number, max_attempts, query, location, max_questions, status, created_at
|
|
76
|
-
)
|
|
77
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'running', ?)
|
|
78
|
-
ON CONFLICT(job_id, attempt_number) DO NOTHING
|
|
79
|
-
`,args:[nr(),e.jobId,e.userId,e.attemptNumber,e.maxAttempts,e.query,e.location,e.maxQuestions,e.startedAt]})}async function Ue(e){return(await a().execute({sql:`
|
|
80
|
-
UPDATE harvest_attempts SET
|
|
81
|
-
browser_provider = ?,
|
|
82
|
-
provider_session_id = ?,
|
|
83
|
-
provider_session_observed_at = COALESCE(provider_session_observed_at, ?),
|
|
84
|
-
provider_lookup_status = CASE
|
|
85
|
-
WHEN provider_lookup_status = 'complete' THEN provider_lookup_status
|
|
86
|
-
ELSE 'pending'
|
|
87
|
-
END
|
|
88
|
-
WHERE job_id = ? AND attempt_number = ? AND status = 'running'
|
|
89
|
-
`,args:[e.browserProvider,e.providerSessionId,e.observedAt,e.jobId,e.attemptNumber]})).rowsAffected===1}async function $e(e){await a().execute({sql:`
|
|
90
|
-
UPDATE harvest_attempts SET
|
|
91
|
-
status = 'finished',
|
|
92
|
-
outcome = ?,
|
|
93
|
-
\u006b\u0065\u0072\u006e\u0065\u006c_session_id = ?,
|
|
94
|
-
question_count = ?,
|
|
95
|
-
duration_ms = ?,
|
|
96
|
-
error = ?,
|
|
97
|
-
will_retry = ?,
|
|
98
|
-
\u006b\u0065\u0072\u006e\u0065\u006c_delete_started = ?,
|
|
99
|
-
\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded = ?,
|
|
100
|
-
\u006b\u0065\u0072\u006e\u0065\u006c_delete_error = ?,
|
|
101
|
-
browser_close_succeeded = ?,
|
|
102
|
-
browser_close_error = ?,
|
|
103
|
-
browser_provider = COALESCE(browser_provider, ?),
|
|
104
|
-
provider_session_id = COALESCE(provider_session_id, ?),
|
|
105
|
-
provider_lookup_status = CASE
|
|
106
|
-
WHEN provider_lookup_status = 'complete' THEN provider_lookup_status
|
|
107
|
-
ELSE COALESCE(?, provider_lookup_status)
|
|
108
|
-
END,
|
|
109
|
-
provider_status = COALESCE(provider_status, ?),
|
|
110
|
-
provider_error_code = CASE
|
|
111
|
-
WHEN provider_reason_source = 'provider_session_api' THEN provider_error_code
|
|
112
|
-
ELSE COALESCE(?, provider_error_code)
|
|
113
|
-
END,
|
|
114
|
-
provider_error_message = CASE
|
|
115
|
-
WHEN provider_reason_source = 'provider_session_api' THEN provider_error_message
|
|
116
|
-
ELSE COALESCE(?, provider_error_message)
|
|
117
|
-
END,
|
|
118
|
-
provider_reason_source = CASE
|
|
119
|
-
WHEN provider_reason_source = 'provider_session_api' THEN provider_reason_source
|
|
120
|
-
ELSE COALESCE(?, provider_reason_source)
|
|
121
|
-
END,
|
|
122
|
-
provider_duration_ms = COALESCE(provider_duration_ms, ?),
|
|
123
|
-
provider_bandwidth_bytes = COALESCE(provider_bandwidth_bytes, ?),
|
|
124
|
-
provider_captcha = COALESCE(provider_captcha, ?),
|
|
125
|
-
provider_checked_at = COALESCE(provider_checked_at, ?),
|
|
126
|
-
provider_detail_json = COALESCE(provider_detail_json, ?),
|
|
127
|
-
provider_disconnect_observed = COALESCE(provider_disconnect_observed, ?),
|
|
128
|
-
provider_disconnect_message = COALESCE(provider_disconnect_message, ?),
|
|
129
|
-
debug_json = ?,
|
|
130
|
-
completed_at = ?
|
|
131
|
-
WHERE job_id = ? AND attempt_number = ?
|
|
132
|
-
`,args:[e.outcome,e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,e.questionCount,e.durationMs,e.error,e.willRetry?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded==null?null:e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,e.browserCloseSucceeded==null?null:e.browserCloseSucceeded?1:0,e.browserCloseError,e.browserProvider??null,e.providerSessionId??null,e.providerLookupStatus??null,e.providerStatus??null,e.providerErrorCode??null,e.providerErrorMessage??null,e.providerReasonSource??null,e.providerDurationMs??null,e.providerBandwidthBytes??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,e.providerCheckedAt??null,e.providerDetail==null?null:Fe(e.providerDetail),e.providerDisconnectObserved==null?null:e.providerDisconnectObserved?1:0,e.providerDisconnectMessage??null,e.debug==null?null:JSON.stringify(e.debug),e.completedAt,e.jobId,e.attemptNumber]})}function Fe(e,r=4096){let t=JSON.stringify(e);return t.length<=r?t:JSON.stringify({truncated:!0})}async function Xr(e){return(await a().execute({sql:`
|
|
133
|
-
UPDATE harvest_attempts SET
|
|
134
|
-
provider_lookup_status = ?,
|
|
135
|
-
provider_status = ?,
|
|
136
|
-
provider_error_code = ?,
|
|
137
|
-
provider_error_message = ?,
|
|
138
|
-
provider_reason_source = ?,
|
|
139
|
-
provider_duration_ms = ?,
|
|
140
|
-
provider_bandwidth_bytes = ?,
|
|
141
|
-
provider_captcha = ?,
|
|
142
|
-
provider_checked_at = ?,
|
|
143
|
-
provider_detail_json = ?
|
|
144
|
-
WHERE id = ?
|
|
145
|
-
AND job_id = ?
|
|
146
|
-
AND user_id = ?
|
|
147
|
-
AND browser_provider = '\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api'
|
|
148
|
-
AND provider_session_id = ?
|
|
149
|
-
AND COALESCE(provider_lookup_status, '') <> 'complete'
|
|
150
|
-
`,args:[e.lookupStatus,e.providerStatus??null,e.errorCode??null,e.errorMessage??null,e.reasonSource??null,e.durationMs??null,e.bandwidthBytes??null,e.captcha==null?null:e.captcha?1:0,e.checkedAt,e.detail==null?null:Fe(e.detail),e.attemptId,e.jobId,e.userId,e.providerSessionId]})).rowsAffected===1}async function Zr(e=25){let r=Math.max(1,Math.min(100,Math.trunc(e)));return(await a().execute({sql:`
|
|
151
|
-
SELECT * FROM harvest_attempts
|
|
152
|
-
WHERE browser_provider = '\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api'
|
|
153
|
-
AND provider_session_id IS NOT NULL
|
|
154
|
-
AND provider_lookup_status IN ('pending', 'temporary_error')
|
|
155
|
-
ORDER BY COALESCE(provider_checked_at, created_at) ASC
|
|
156
|
-
LIMIT ?
|
|
157
|
-
`,args:[r]})).rows.map(n=>qe(n))}async function We(e,r){return(r==null?await a().execute({sql:"SELECT * FROM harvest_attempts WHERE job_id = ? ORDER BY attempt_number ASC",args:[e]}):await a().execute({sql:`
|
|
158
|
-
SELECT ha.*
|
|
159
|
-
FROM harvest_attempts ha
|
|
160
|
-
INNER JOIN jobs j ON j.id = ha.job_id
|
|
161
|
-
WHERE ha.job_id = ? AND j.user_id = ?
|
|
162
|
-
ORDER BY ha.attempt_number ASC
|
|
163
|
-
`,args:[e,r]})).rows.map(n=>qe(n))}import{createHash as it}from"crypto";var ir=1e9;function M(e){return e==="timeout"?"timed_out":e==="request_aborted"?"cancelled":e==="browser_session_interrupted"?"interrupted":e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"||e==="location_mismatch"||e==="error"?"failed":"succeeded"}function sr(e){return e==="\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061":"\u006b\u0065\u0072\u006e\u0065\u006c"}function Be(e,r,t=0){let n=new Map,s=null;return async i=>{if(e)try{if(i.type==="started"){let l=t+i.attemptNumber;if(l>1&&!s){let v=(await fe(e))?.attempts.at(-1);s=v?.id==null?null:String(v.id)}let d=await be({runId:e,attemptNumber:l,method:i.method,provider:i.provider,parentAttemptId:l===1?null:s,edgeKind:l===1?null:t>0&&i.attemptNumber===1?"durable_resume":i.retryKind==="provider_fallback"?"provider_fallback":"same_method_retry",retryKind:l===1?null:t>0&&i.attemptNumber===1?"durable_resume":i.retryKind,retryReason:l===1?null:t>0&&i.attemptNumber===1?"durable_worker_resume":i.retryReason,retryOwner:l===1?null:t>0&&i.attemptNumber===1?"durable_reconciler":"harvest",normalizedFlags:{maxAttempts:i.maxAttempts,maxQuestions:i.maxQuestions,hasLocation:!!i.location,plannedProvider:i.provider,planReason:i.planReason??null,planTimeoutMs:i.planTimeoutMs??null},startedAt:i.startedAt}),u=await D({attemptId:d,provider:i.provider,providerProduct:i.provider==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:i.provider==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?R("https://www.google.com/search"):null,accountScope:r,method:i.method,providerStatus:"started"});n.set(i.attemptNumber,{attemptId:d,receiptId:u.id,plannedProvider:i.provider,observedProvider:null,providerMismatch:!1,method:i.method,startedAtMs:Date.parse(i.startedAt),billingMode:null}),s=d;return}let o=n.get(i.attemptNumber);if(!o)throw new Error(`Missing operation attempt state for attempt ${i.attemptNumber}`);if(i.type==="provider_session_started"){let l=sr(i.provider);if(o.observedProvider=l,o.billingMode=i.billingMode??null,l!==o.plannedProvider){o.providerMismatch=!0,await k({receiptId:o.receiptId,costState:"unavailable",providerStatus:"provider_mismatch",errorCode:"provider_mismatch",errorMessage:`Planned ${o.plannedProvider}; observed ${l}`,finalizedAt:i.observedAt});let d=l==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",u=await D({attemptId:o.attemptId,provider:l,providerProduct:l==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:l==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?R("https://www.google.com/search"):null,accountScope:r,method:d,providerStatus:"observed_provider_mismatch"});o.receiptId=u.id,o.method=d}await ve(o.receiptId,i.providerSessionId);return}if(await ge({attemptId:o.attemptId,status:o.providerMismatch?"failed":M(i.outcome),errorCode:o.providerMismatch?"provider_mismatch":M(i.outcome)==="succeeded"?null:i.outcome,errorMessage:i.error,durationMs:i.durationMs,endedAt:i.completedAt}),(o.observedProvider??o.plannedProvider)==="\u006b\u0065\u0072\u006e\u0065\u006c"){let l=o.billingMode==="headful",d=o.billingMode==="headless"||o.billingMode==="headful"?Z(i.durationMs,l):null;await k({receiptId:o.receiptId,costState:"estimated",units:i.durationMs/1e3,unitType:"wall_open_second",amountUsdNanos:d==null?null:Math.round(d*ir),providerProduct:"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:o.billingMode,billableEvent:"provider_active_gb_second",measurementSource:"wall_open_time_proxy",rateSource:d!=null?`${X};\u006b\u0065\u0072\u006e\u0065\u006c_usd_per_second:${V}`:null,providerStatus:o.providerMismatch?"provider_mismatch":i.outcome,providerDurationMs:i.durationMs,errorCode:o.providerMismatch?"provider_mismatch":M(i.outcome)==="succeeded"?null:i.outcome,errorMessage:i.error,finalizedAt:i.completedAt})}}catch(o){console.warn(JSON.stringify({event:"operation_attempt_telemetry_failed",operation_run_id:e,attempt_number:i.attemptNumber,message:o instanceof Error?o.message:String(o)}))}}}function ze(e,r,t=null,n="production-default",s=0){let i=Be(t,n,s);return async o=>{if(await i(o),o.type==="started"){await Je({jobId:e,userId:r,attemptNumber:o.attemptNumber,maxAttempts:o.maxAttempts,query:o.query,location:o.location,maxQuestions:o.maxQuestions,startedAt:o.startedAt});return}if(o.type==="provider_session_started"){await Ue({jobId:e,attemptNumber:o.attemptNumber,browserProvider:o.provider,providerSessionId:o.providerSessionId,observedAt:o.observedAt});return}await $e({jobId:e,attemptNumber:o.attemptNumber,outcome:o.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:o.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:o.questionCount,durationMs:o.durationMs,error:o.error,willRetry:o.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:o.cleanup.browserCloseSucceeded,browserCloseError:o.cleanup.browserCloseError,browserProvider:o.cleanup.provider??null,providerSessionId:o.cleanup.providerSessionId??null,providerLookupStatus:o.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:o.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:o.cleanup.providerDisconnectMessage??null,providerReasonSource:o.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:o.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:o.cleanup.providerDisconnectMessage??null,debug:o.debug,completedAt:o.completedAt})}}function ar(e){return e instanceof Error?e.message:String(e)}function lr(e,r){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(r)}function dr(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function ur(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function cr(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function pr(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function mr(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function L(e){let r=ar(e);return e instanceof G?{error_code:"request_aborted",error_type:"request_aborted",message:m("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof K||dr(r)?{error_code:"captcha_exhausted",error_type:"captcha",message:m("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:F(e,r)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:m("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof Q?{error_code:"location_mismatch",error_type:"location_mismatch",message:m("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:ur(r)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:m("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:cr(r)?{error_code:"proxy_unavailable",error_type:"connection",message:m("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:mr(r)?{error_code:"browser_session_interrupted",error_type:"extraction",message:m("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:lr(e,r)?{error_code:"harvest_timeout",error_type:"timeout",message:m("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:pr(r)?{error_code:"service_unavailable",error_type:"service_unavailable",message:N,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:N,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function bt(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function Ke(e,r={}){let t=A({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:r.retryAfterSeconds,chargeStatus:r.chargeStatus,details:r.details});return{error:t.message,...t}}function Ge(e,r={}){let{error:t,...n}=Ke(e,r);return n}function gt(e,r={}){let t=L(e);return{body:Ke(t,r),status:t.httpStatus}}import{randomUUID as Qe}from"crypto";var _r=3e4;async function Ye(e,r,t){if(!e.receiptJson)return;let n=C(e.id);if((await b(n,e.userId,"paa_callback",e.jobId))?.phase==="published")return;if(t){if(await ne(t,{status:"succeeded",json:e.receiptJson}))return;await I(t)}let i=await oe(n,e.userId);i&&await ie(i,{status:"unknown",json:e.receiptJson},P(e,r,"accepted"))}function br(e){return!!(e.payloadJson&&e.payloadSha256===h(e.payloadJson)&&e.sendNonce&&e.sendToken&&e.httpStatus&&e.httpStatus>=200&&e.httpStatus<300&&e.receiptJson===f({deliveryId:e.id,idempotencyKey:e.id,payloadSha256:e.payloadSha256,httpStatus:e.httpStatus}))}async function Ve(e){let r=e.record,t,n=!1;try{if(r=await y(r.id,r.userId)??r,r.state==="accepted")return br(r)?(await Ye(r,e.domain).catch(()=>{}),p(r)):{state:"unavailable",deliveryId:r.id};if(["sending","unknown","expired","unavailable"].includes(r.state))return p(r);if(r.expiresAtMs<=r.nowMs)return await w(r,"expired"),p(await y(r.id,r.userId));if(r.payloadJson||(r=await Se(r,await e.compose(),e.domain)),!r.payloadJson||r.payloadSha256!==h(r.payloadJson))throw Error("Delivery payload conflicts");let s=P(r,e.domain,"frozen"),i=C(r.id),o={version:1,purpose:"outbound-callback",request:ye(r),payloadSha256:r.payloadSha256},l=await ae({id:i,userId:r.userId,ownerScope:`user:${r.userId}`,tool:"paa_callback",jobId:r.jobId,idempotencyKey:r.id,requestFingerprint:h(f(o)),normalizedFlags:o},Qe(),_r,s);if(l.kind!=="claimed"){let u=await b(i,r.userId,"paa_callback",r.jobId);if(u&&["effects_started","unknown","published"].includes(u.phase)){await T(r),u.phase==="published"&&await w(r,"unavailable",s);let _=await y(r.id,r.userId);return _?p(_):{state:"unknown",deliveryId:r.id}}return{state:"pending",deliveryId:r.id,expiresAtMs:r.expiresAtMs}}if(t=l.claim,!await e.validate(r.url))return p(r);if(!await le(t,s))throw Error("Delivery effects start is unconfirmed");if(n=!0,r=await he(r,t,Qe(),e.domain),!await Ae(r,t,e.domain))throw Error("Delivery authority was lost before send");let d=await fetch(r.url,{method:"POST",headers:{"content-type":"application/json","idempotency-key":r.id},body:r.payloadJson,redirect:"error",signal:AbortSignal.timeout(1e4)});if(!d.ok||d.status<200||d.status>=300)throw Error("Recipient acceptance is unconfirmed");return r=await we(r,d.status),await Ye(r,e.domain,t).catch(()=>{}),p(r)}catch(s){if(s instanceof Ee&&await w(r,"unavailable",e.domain).catch(()=>{}),t){let o=await b(t.runId,t.userId,"paa_callback",r.jobId).catch(()=>null);(n||o?.phase==="effects_started"||o?.phase==="unknown")&&await I(t).catch(()=>{})}r.sendNonce?await xe(r).catch(()=>{}):await T(r).catch(()=>{});let i=await y(r.id,r.userId).catch(()=>null);return i?p(i):{state:"unknown",deliveryId:r.id}}finally{t&&!n&&await se(t).catch(()=>{})}}async function H(e,r){let t=await Ie(e,r);return t.kind==="none"?{state:"none"}:t.kind==="unavailable"?{state:"unavailable"}:t.kind==="pending"?(await p(t.record),{state:"pending",deliveryId:t.record.id,expiresAtMs:t.record.expiresAtMs}):Ve({record:t.record,domain:t.domain,validate:async n=>{let s=await _e(n,{field:"callback_url",requireHttps:!0});return!s.error&&s.parsed?.href===n&&!s.parsed.username&&!s.parsed.password},compose:async()=>{let n=(await Oe(t.job,!1)).body;return{...n,...n.result?{result:te(n.result)}:{},...t.record.profile==="paa-queue"?{attempts:re(await We(t.job.id,t.job.user_id))}:{}}}})}async function Rt(e=25){let r=0,t=0,n=await Ne(e);for(let s of n)try{(await H(s.jobId,s.userId)).state==="accepted"?r++:t++}catch{t++}return{checked:n.length,accepted:r,pending:t}}var Xe=2,q=0;async function Ze(){let e=await Me(),r=e?void 0:await Pe(),t=e??(r?await c(r.id,Number(r.user_id)):void 0);if(!t)return{claimed:!1};let n=Date.now();q++;try{let s=await Le(t,{\u006b\u0065\u0072\u006e\u0065\u006cApiKey:B(),apifyToken:process.env.APIFY_API_TOKEN?.trim(),onAttemptEvent:ze(t.id,t.user_id,`harvest:${t.id}`),withCostContext:Y,classifyFailure:i=>{let o=L(i);return{publicError:Ge(o),httpStatus:o.httpStatus,terminalStatus:o.terminalStatus}}});return s.completed&&t.callback_url&&await H(t.id,t.user_id).catch(()=>{}),{claimed:!0,jobId:t.id,completed:s.completed,durationMs:Date.now()-n}}finally{q--}}async function Ut(e){let r=[];for(let t=0;t<e.maxJobs&&!(Date.now()>=e.deadlineMs);t++){let n=await Ze();if(r.push(n),!n.claimed)break}return r}function $t(){setInterval(async()=>{q>=Xe||Ze()},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${Xe} concurrent`)}export{Ar as a,wr as b,xr as c,Nr as d,Ir as e,Dr as f,kr as g,Or as h,Xr as i,Zr as j,We as k,L as l,bt as m,Ge as n,gt as o,H as p,Rt as q,Tr as r,Te as s,Gr as t,or as u,ze as v,Ze as w,Ut as x,$t as y};
|
package/dist/chunk-JHDD6R3O.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
var n=class extends Error{constructor(){super("Current operation wallet claim and matching durable plan are required"),this.name="WalletClaimLostError"}},t=class extends Error{constructor(){super("Named operation debit plan conflicts with durable billing history"),this.name="DebitPlanConflictError"}};function i(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0||!Number.isSafeInteger(e.amountMc)||e.amountMc<=0||!e.key||e.key.length>500||!e.operation||!e.description||e.description.length>500)throw new t;return{sql:`EXISTS (SELECT 1 FROM billing_debits d WHERE d.idempotency_key=? AND d.user_id=?
|
|
2
|
-
AND d.amount_mc=? AND d.operation=? AND d.description=? AND d.status='applied'
|
|
3
|
-
AND NOT EXISTS (SELECT 1 FROM billing_events b WHERE b.user_id=d.user_id
|
|
4
|
-
AND b.billing_class='wallet_hold_settlement'
|
|
5
|
-
AND json_extract(b.metadata,'$.debitIdempotencyKey')=d.idempotency_key))`,args:[e.key,e.userId,e.amountMc,e.operation,e.description]}}function o(e,r){if(!/^[a-z_]+\.[a-z_]+$/.test(e)||!/^json_extract\([a-z_]+\.[a-z_]+,'\$\.hold\.key'\)$/.test(r))throw new Error("Invalid stored hold selector");return`EXISTS (SELECT 1 FROM billing_debits d WHERE d.user_id=${e}
|
|
6
|
-
AND d.idempotency_key=${r} AND d.status='applied'
|
|
7
|
-
AND NOT EXISTS (SELECT 1 FROM billing_events b WHERE b.user_id=d.user_id
|
|
8
|
-
AND b.billing_class='wallet_hold_settlement' AND json_extract(b.metadata,'$.debitIdempotencyKey')=d.idempotency_key))`}export{n as a,t as b,i as c,o as d};
|
package/dist/chunk-JMZIDHYQ.js
DELETED
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import{a as D}from"./chunk-OLYPLX5C.js";import{b as R}from"./chunk-B3JRL45T.js";import{c as P}from"./chunk-OILOHQNL.js";import{createWriteStream as H,mkdirSync as W,rmSync as N}from"fs";import{join as A,extname as B,basename as z}from"path";import{pipeline as O}from"stream/promises";import{Readable as I,Transform as q}from"stream";var X=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],G=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),K=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),V=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function Z(r){let e=r.toLowerCase();return X.some(t=>e.includes(t))}function J(r){return r.startsWith("data:")}function U(r){try{let e=B(new URL(r).pathname).toLowerCase();if(G.has(e))return"image";if(K.has(e))return"video";if(V.has(e))return"audio"}catch{}return null}function L(r){let e=r.toLowerCase();return e.startsWith("image/")?"image":e.startsWith("video/")?"video":e.startsWith("audio/")?"audio":null}function Q(r,e){if(!r||J(r))return null;try{return new URL(r,e).href}catch{return null}}function C(r,e){try{let t=new URL(r);return z(t.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${e}`}catch{return`asset-${e}`}}function k(r,e){let t=r?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return t?t.slice(0,e):null}function Y(r){let e=r.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return e?{width:Number(e[1]),height:Number(e[2])}:null}function ee(r){try{let e=new URL(r);e.hash="",e.pathname=e.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let t of["w","width","h","height","resize"])e.searchParams.delete(t);return e.searchParams.sort(),e.href}catch{return r}}function T(r){let e=(r.width??0)*(r.height??0),t=r.discoveryMethods.some(f=>f.includes("lightbox"))?1e12:0,u=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(r.url)?0:1e11;return t+u+e}function S(r,e){let t=r.get(e.url);if(!t){r.set(e.url,e);return}t.discoveryMethods=[...new Set([...t.discoveryMethods,...e.discoveryMethods])],t.altTexts=[...new Set([...t.altTexts,...e.altTexts])],t.contexts=[...new Set([...t.contexts,...e.contexts])],!t.type&&e.type&&(t.type=e.type),(e.width??0)*(e.height??0)>(t.width??0)*(t.height??0)&&(t.width=e.width,t.height=e.height)}function te(r,e){let t=new Map,u=(l,s,a=null,h,i,c)=>{let n=l?Q(l.trim().replace(/&/g,"&"),e):null;if(!n||Z(n))return;let d=c??Y(n);S(t,{url:n,type:a??U(n),discoveryMethods:[s],altTexts:h?[h]:[],contexts:i?[i]:[],width:d?.width??null,height:d?.height??null})},f=(l,s,a,h)=>{for(let i of(l??"").split(",")){let[c,n]=i.trim().split(/\s+/);if(!c)continue;let d=n?.match(/^(\d+)w$/)?.[1];u(c,s,a,h,null,d?{width:Number(d),height:0}:null)}},m=(l,s="")=>{let a=l.$;for(let i of l.images){let c=i.attributes,n=k(i.alt,500),d=i.width&&i.height?{width:i.width,height:i.height}:null;for(let g of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=c[g];u(y,`static-${g}${s}`,"image",n,null,d)}f(c.srcset,`static-srcset${s}`,"image",n),f(c["data-srcset"],`static-data-srcset${s}`,"image",n)}a("source").each((i,c)=>{let n=a(c),d=n.attr("type")??"",g=d.startsWith("video/")?"video":d.startsWith("audio/")?"audio":"image";u(n.attr("src"),`static-source${s}`,g),f(n.attr("srcset"),`static-source-srcset${s}`,g)}),a("video,audio").each((i,c)=>{let n=a(c),d=c.tagName.toLowerCase();u(n.attr("src"),`static-${d}${s}`,d),d==="video"&&u(n.attr("poster"),`static-video-poster${s}`,"image")}),a("meta").each((i,c)=>{let n=a(c),d=(n.attr("property")??n.attr("name")??"").toLowerCase();(d==="og:image"||d==="og:image:url"||d==="twitter:image"||d==="twitter:image:src")&&u(n.attr("content"),`static-${d}${s}`,"image")}),a("link[href]").each((i,c)=>{let n=a(c);(n.attr("rel")??"").toLowerCase().split(/\s+/).some(g=>g==="icon"||g==="apple-touch-icon")&&u(n.attr("href"),`static-icon${s}`,"image")}),a("svg image").each((i,c)=>{let n=a(c);u(n.attr("href")??n.attr("xlink:href"),`static-svg-image${s}`,"image")}),a("a[href]").has("img").each((i,c)=>{let n=a(c),d=n.attr("href");if(d&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${n.attr("class")??""} ${n.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(d))){let g=k(n.find("img").first().attr("alt"),500);u(d,`static-lightbox-target${s}`,"image",g)}});let h=a("style").toArray().map(i=>a(i).text());a("[style]").each((i,c)=>{h.push(a(c).attr("style")??"")});for(let i of h.join(`
|
|
2
|
-
`).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|"|�?34;|'|�?39;)?([^"')&]+)(?:["']|"|�?34;|'|�?39;)?\s*\)/gi))u(i[1],`static-css-url${s}`,"image");for(let i of l.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))u(i[1],`static-bare-url${s}`)},o=typeof r=="string"?D(r):r;return m(o),o.$("script").each((l,s)=>{let a=o.$(s).text(),h=a.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,i=>i.slice(1)).replace(/\\\//g,"/");h!==a&&/<(?:img|source|video|audio)\b/i.test(h)&&m(D(h),"-json-unescaped")}),[...t.values()]}function re(r,e,t=100){let u=new Map;for(let m of r)S(u,m);for(let m of e?.assets??[])S(u,{...m,type:m.type});let f=new Map;for(let m of u.values()){let o=ee(m.url),l=f.get(o)??[];l.push(m),f.set(o,l)}return[...f.values()].map(m=>{let o=[...m].sort((s,a)=>T(a)-T(s)||s.url.localeCompare(a.url)),l={...o[0]};return l.discoveryMethods=[...new Set(m.flatMap(s=>s.discoveryMethods))],l.altTexts=[...new Set(m.flatMap(s=>s.altTexts))],l.contexts=[...new Set(m.flatMap(s=>s.contexts))],{...l,variants:o.slice(1).map(s=>s.url)}}).sort((m,o)=>T(o)-T(m)||m.url.localeCompare(o.url)).slice(0,Math.max(1,t))}async function ne(r,e,t,u={}){let f=Math.max(1,Math.min(u.maxBytes??52428800,104857600)),m=r,o=null;for(let g=0;g<=5;g++){let y=await R(m,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(o=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),o.status>=300&&o.status<400){let v=o.headers.get("location");if(!v)throw new Error(`HTTP ${o.status} redirect did not include Location`);m=new URL(v,y.parsed.href).href,o=null;continue}break}if(!o)throw new Error("Media download exceeded five redirects");if(!o.ok)throw new Error(`HTTP ${o.status}`);if(!o.body)throw new Error("Empty response body");let l=o.headers.get("content-type")?.split(";")[0].trim()??null;if(u.expectedType&&(!l||L(l)!==u.expectedType))throw new Error(`Expected ${u.expectedType} content but received ${l??"no content-type"}`);let s=Number(o.headers.get("content-length")??0);if(Number.isFinite(s)&&s>f)throw new Error(`Media asset exceeds ${f} byte limit`);let a=A(e,t);if(l&&!B(t)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[l];y&&(a=a+y)}let h=H(a);await new Promise((g,y)=>{h.once("open",()=>g()),h.once("error",y)});let i=0,c=new q({transform(g,y,v){let b=g.length;if(i+b>f){v(new Error(`Media asset exceeds ${f} byte limit`));return}if(u.consumeBytes&&!u.consumeBytes(b)){v(new Error("Media export total byte limit exceeded"));return}i+=b,v(null,g)}});try{await O(I.fromWeb(o.body),c,h)}catch(g){throw h.destroy(),N(a,{force:!0}),g}let{statSync:n}=await import("fs"),d=n(a).size;return{savedPath:a,sizeBytes:d,mimeType:l}}async function ue(r,e,t={}){let u=t.types??["image","video","audio"],f=new Set(u),m=te(r,e),o=Math.max(1,Math.min(t.maxAssets??100,250)),l=re(m,t.rendered,2e3),s=l.slice(0,o),a=new Set([...m.map(p=>p.url),...(t.rendered?.assets??[]).map(p=>p.url)]).size,h=s.flatMap(p=>{let w=p.type??U(p.url);return w&&f.has(w)?[{...p,type:w}]:[]}),i=a-h.length,c=l.length>o,n=t.rendered?.exhausted===!0&&!c,d=c?"asset_limit":t.rendered?.stopReason??"render_unavailable",g=c?[`Media discovery found ${l.length} responsive families; only the requested ${o} records were retained.`]:t.rendered?t.rendered.exhausted?[]:[`Rendered media discovery stopped with ${t.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:e,staticFound:m.length,renderedFound:t.rendered?.assets.length??0,filteredCount:i,totalFound:a,completeness:n?"complete":"partial",exhausted:n,stopReason:d,scrollRounds:t.rendered?.scrollRounds??0,warnings:g,artifact:null};if(t.outputDir===null)return{...y,outputDir:null,assets:h.map((p,w)=>({...p,mimeType:null,filename:C(p.url,w),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let v=(()=>{try{return new URL(e).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),b=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),$=t.outputDir??A(P(),"media",`${b}-${v}`);W($,{recursive:!0});let M=[];return await Promise.allSettled(h.map(async(p,w)=>{let E=C(p.url,w);try{let{savedPath:x,sizeBytes:j,mimeType:_}=await ne(p.url,$,E),F=_?L(_)??p.type:p.type;M.push({...p,type:F,mimeType:_,filename:z(x),savedPath:x,sizeBytes:j,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(x){M.push({...p,mimeType:null,filename:E,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:x instanceof Error?x.message:"media_download_failed",inlinePreview:null})}})),M.sort((p,w)=>p.savedPath&&!w.savedPath?-1:!p.savedPath&&w.savedPath?1:p.url.localeCompare(w.url)),{...y,outputDir:$,assets:M}}export{L as a,te as b,ne as c,ue as d};
|
package/dist/chunk-K3VDGJNK.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{a as w,b as u,c as _,d as l,e as h,m as g,p}from"./chunk-7SR7MEU5.js";import{k as c,m}from"./chunk-ANQ4JJVI.js";import{b as I,c as s}from"./chunk-4ZVO6GX4.js";async function R(e){let r=s(),n=!r?.operationRunId,o=r?.operationRunId??(await c({userId:r?.userId??null,ownerScope:r?.userId!=null?`user:${r.userId}`:"system:unscoped",tool:e.tool,normalizedFlags:{subOperation:r?.subOp??null},deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,a=n?null:await p(o),i=a?.attempts.at(-1),t=e.parentAttemptId??(e.retryKind&&i?String(i.id):r?.operationRootAttemptId),b=e.edgeKind??(e.retryKind==="same_method"?"same_method_retry":e.retryKind==="provider_fallback"?"provider_fallback":void 0),d=t?await _({runId:o,parentAttemptId:t,edgeKind:b??"parallel_child",method:e.method,provider:e.provider,retryKind:e.retryKind,retryReason:e.retryReason}):await u({runId:o,attemptNumber:(a?.attempts.length??0)+1,method:e.method,provider:e.provider,...i?{parentAttemptId:String(i.id),edgeKind:b??"next_phase",retryKind:e.retryKind??null,retryReason:e.retryReason??null}:{}}),y;try{y=(await h({attemptId:d,provider:e.provider,providerProduct:e.providerProduct,billingTier:e.billingTier,accountScope:r?.operationAccountScope??"production-default",method:e.method,providerStatus:"started"})).id}catch(f){throw await w({tool:e.tool,method:e.method,phase:"pending_receipt_create",errorCode:f instanceof Error?f.name:"UnknownError",runId:o,attemptId:d}).catch(()=>{}),await l({attemptId:d,status:"failed",errorCode:"pending_receipt_create_failed",durationMs:0}).catch(()=>{}),f}return{runId:o,attemptId:d,receiptId:y,ownsRun:n,startedAtMs:Date.now()}}async function S(e,r,n){await l({attemptId:e.attemptId,status:r,errorCode:n??null,durationMs:Date.now()-e.startedAtMs}),e.ownsRun&&await m({runId:e.runId,status:r,failureClass:n??null})}async function T(e,r){let n=s(),o=null,a=Date.now();if(n?.operationRunId&&n.operationRootAttemptId)try{o=await _({runId:n.operationRunId,parentAttemptId:n.operationRootAttemptId,edgeKind:"parallel_child",method:e})}catch(t){console.warn(JSON.stringify({event:"local_cost_phase_start_failed",method:e,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await r();return i="succeeded",t}finally{o&&await l({attemptId:o,status:i,durationMs:Date.now()-a,errorCode:i==="failed"?"local_phase_failed":null}).catch(t=>console.warn(JSON.stringify({event:"local_cost_phase_finish_failed",method:e,message:t instanceof Error?t.message:String(t)})))}}async function M(e,r){let n=s()?.operationRunId;n&&await g({runId:n,billingTable:"billing_debits",billingKey:e,relation:"debit",amountMc:r}).catch(o=>console.warn(JSON.stringify({event:"cost_billing_link_failed",message:o instanceof Error?o.message:String(o)})))}async function K(e,r){let n=s();if(n?.operationRunId)return I({...n,op:e.op,userId:e.userId},r);let o=null,a=null;try{o=(await c({userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,e.billingDebitKey&&await g({runId:o,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}).catch(t=>console.warn(JSON.stringify({event:"cost_billing_link_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))),a=await u({runId:o,attemptNumber:1,method:`${e.tool}_root`,normalizedFlags:e.normalizedFlags})}catch(t){console.warn(JSON.stringify({event:"logical_cost_run_start_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await I({...n,op:e.op,userId:e.userId,operationRunId:o,operationRootAttemptId:a},r);return i=t instanceof Response&&t.status>=400?"failed":"succeeded",t}finally{o&&a?await v({runId:o,rootAttemptId:a,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(t=>console.warn(JSON.stringify({event:"logical_cost_run_finish_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))):o&&await m({runId:o,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(()=>{})}}async function O(e){let r=await c({...e.jobId?{id:`extract-url:${e.jobId}`,idempotencyKey:`job:${e.jobId}`,requestFingerprint:e.jobId,jobId:e.jobId}:{},userId:e.userId,ownerScope:`user:${e.userId}`,tool:"extract_url",normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null}),n=r.replayed?await p(r.id):null,o=n?.attempts[0]?.id==null?await u({runId:r.id,attemptNumber:1,method:"extract_url_root",normalizedFlags:e.normalizedFlags}):String(n.attempts[0].id);return e.billingDebitKey&&await g({runId:r.id,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}),{runId:r.id,rootAttemptId:o}}async function v(e){let r=await p(e.runId),n=r?.attempts.find(o=>String(o.id)===e.rootAttemptId);n?.status==="running"&&await l({attemptId:e.rootAttemptId,status:e.status,errorCode:e.failureClass??null,durationMs:Math.max(0,Date.now()-Date.parse(String(n.started_at)))}),r?.run.status==="running"&&await m({runId:e.runId,status:e.status,failureClass:e.failureClass??null})}export{R as a,S as b,T as c,M as d,K as e,O as f,v as g};
|
package/dist/chunk-KBZY2OGP.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import{c as y}from"./chunk-JHDD6R3O.js";var b="strftime('%Y-%m-%dT%H:%M:%fZ','now')",c="julianday('now')";var L=20,S="(json_extract(options,'$.executionOwner')='direct' AND json_extract(options,'$.serpOnly') IS NOT 1)",I=`NOT COALESCE(${S},0) AND json_extract(options,'$.syncExecutionProtocol') IS NOT 1 AND json_extract(options,'$.queueExecutionProtocol') IS NULL`,v=`${S} AND julianday(json_extract(options,'$.paaLeaseExpiresAt')) > ${c}`,O=`${S} AND (
|
|
2
|
-
(json_extract(options,'$.paaLeaseExpiresAt') IS NOT NULL AND COALESCE(julianday(json_extract(options,'$.paaLeaseExpiresAt')),0) <= ${c})
|
|
3
|
-
OR (json_extract(options,'$.paaLeaseExpiresAt') IS NULL AND julianday(started_at) <= julianday('now','-15 minutes'))
|
|
4
|
-
)`;function h(e){return{jobId:e.id,userId:e.user_id,leaseToken:typeof e.options.paaLeaseToken=="string"?e.options.paaLeaseToken:null,recoveryCount:Number(e.options.paaRecoveryCount??0),resultJson:e.result==null?null:JSON.stringify(e.result)}}function T(e,t){return!t||t.jobId!==e||!Number.isSafeInteger(t.userId)||t.userId<=0||!Number.isSafeInteger(t.recoveryCount)||t.recoveryCount<0||t.leaseToken!==null&&(typeof t.leaseToken!="string"||t.leaseToken.length===0)?{sql:"0",args:[]}:{sql:`user_id=? AND json_extract(options,'$.paaLeaseToken') IS ?
|
|
5
|
-
AND COALESCE(json_extract(options,'$.paaRecoveryCount'),0)=? AND json(result) IS json(?)`,args:[t.userId,t.leaseToken,t.recoveryCount,t.resultJson]}}function q(e,t,n){let a={userId:e.user_id,tool:"harvest_paa",jobId:e.id,sql:"0",args:[]},r=e.options.requestFingerprint;if(e.options.executionProtocol!==1||e.options.executionOwner!=="direct"||e.options.serpOnly===!0||typeof r!="string"||!/^[a-f0-9]{64}$/.test(r)||!t||n.userId!==e.user_id||n.key!==e.options.billingDebitKey||n.amountMc!==e.options.billingHoldMc)return a;let i=y(n);return{...a,sql:`EXISTS (SELECT 1 FROM jobs j WHERE j.id=? AND j.user_id=? AND j.query=? AND j.status='running'
|
|
6
|
-
AND json_extract(j.options,'$.executionOwner')='direct' AND json_extract(j.options,'$.serpOnly') IS NOT 1
|
|
7
|
-
AND json_extract(j.options,'$.executionProtocol')=1 AND json_extract(j.options,'$.requestFingerprint')=?
|
|
8
|
-
AND json_extract(j.options,'$.paaLeaseToken')=?
|
|
9
|
-
AND julianday(json_extract(j.options,'$.paaLeaseExpiresAt')) > ${c}
|
|
10
|
-
AND json_extract(j.options,'$.billingDebitKey')=? AND json_extract(j.options,'$.billingHoldMc')=?
|
|
11
|
-
AND json_extract(j.options,'$.billingSettledAt') IS NULL AND (${i.sql}))`,args:[e.id,e.user_id,e.query,r,t,n.key,n.amountMc,...i.args]}}import{createHash as C}from"crypto";var A=["observed_present","observed_absent","incomplete","not_requested","unknown","unsupported"],x=["provider_transport","invalid_payload","serp_readiness","query_integrity","pagination","optional_feature_parsing","unknown"],f=class extends Error{stage;failureCode;constructor(t,n,a){super(a),this.name="SerpExtractionError",this.stage=t,this.failureCode=n}};function d(e){return C("sha256").update(e).digest("hex")}function E(e){return e.normalize("NFKC").toLocaleLowerCase("en-US").split(/[^\p{L}\p{N}]+/u).filter(Boolean).slice(0,64)}function k(e){try{let t=new URL(e);return t.hash="",t.pathname.length>1&&(t.pathname=t.pathname.replace(/\/+$/,"")),t.toString()}catch{return e.trim()}}function D(e,t){let n=e===2?[1,2]:[1],a=t.filter(o=>o.attempted).map(o=>o.page),r=t.filter(o=>o.captured&&o.resultCount>0).map(o=>o.page),i=t.find(o=>o.failureCode);return{status:n.every(o=>r.includes(o))?"complete":"incomplete",requestedPages:n,attemptedPages:a,capturedPages:r,pageResultCounts:n.map(o=>({page:o,resultCount:t.find(g=>g.page===o)?.resultCount??0})),providerRequestCount:a.length,evidence:t,failureCode:i?.failureCode??null}}function F(e){return e.requested?e.moduleObserved?!e.rawShapeValid||e.rawCount>0&&e.mappedCount===0?"incomplete":e.mappedCount>0?"observed_present":"observed_absent":"unknown":"not_requested"}function $(e){let t=d(e.query),n=e.finalQuery==null?null:d(e.finalQuery),a=e.locationResolutionSource==="unresolved";return{status:n!==null&&n!==t?"query_mismatch":a?"unresolved_location":"matched",queryHash:t,normalizedQueryTokens:E(e.query),canonicalLocation:e.canonicalLocation??null,locationResolutionSource:e.locationResolutionSource,locationAmbiguousCandidates:Math.max(0,Math.trunc(e.locationAmbiguousCandidates??0)),uulePresent:!!e.uule,uuleHash:e.uule?d(e.uule):null,outboundProviderMethod:e.outboundProviderMethod,finalQueryHash:n}}function Q(e){return d(JSON.stringify(Object.fromEntries(Object.entries(e).filter(([t,n])=>t!=="requestFingerprint"&&n!==void 0).sort(([t],[n])=>t.localeCompare(n)))))}function p(e){return e&&typeof e=="object"&&!Array.isArray(e)?e:{}}function m(e){return typeof e=="number"&&Number.isFinite(e)&&e>=0?Math.trunc(e):0}function u(e,t){return typeof e=="string"&&e.length<=t?e:null}function H(e){let t=p(e),n=p(t.pagination),a=p(t.features),r=p(t.queryIntegrity),i=s=>Array.isArray(s)?s.filter(l=>l===1||l===2).slice(0,2):[],o=i(n.requestedPages);if(o.length===0)return null;let g=new Set(A),_=new Set(x);return{pagination:{status:n.status==="complete"?"complete":"incomplete",requestedPages:o,attemptedPages:i(n.attemptedPages),capturedPages:i(n.capturedPages),pageResultCounts:Array.isArray(n.pageResultCounts)?n.pageResultCounts.slice(0,2).map(s=>{let l=p(s);return{page:l.page===2?2:1,resultCount:m(l.resultCount)}}):[],providerRequestCount:m(n.providerRequestCount),failureCode:u(n.failureCode,100)},features:Object.fromEntries(["localPack","forums","videos","aiOverview","whatPeopleSaying","aiMode","paaPreview","entityIds"].map(s=>[s,g.has(String(a[s]))?a[s]:"unknown"])),queryIntegrity:{status:["matched","unresolved_location","query_mismatch"].includes(String(r.status))?r.status:"query_mismatch",queryHash:u(r.queryHash,64)??"",normalizedQueryTokens:Array.isArray(r.normalizedQueryTokens)?r.normalizedQueryTokens.slice(0,64).flatMap(s=>typeof s=="string"?[s.slice(0,100)]:[]):[],canonicalLocation:u(r.canonicalLocation,500),locationResolutionSource:u(r.locationResolutionSource,100)??"unknown",locationAmbiguousCandidates:m(r.locationAmbiguousCandidates),uulePresent:r.uulePresent===!0,uuleHash:u(r.uuleHash,64),outboundProviderMethod:"search",finalQueryHash:u(r.finalQueryHash,64)},failureStage:_.has(String(t.failureStage))?t.failureStage:null}}export{b as a,L as b,S as c,I as d,v as e,O as f,h as g,T as h,q as i,f as j,k,D as l,F as m,$ as n,Q as o,H as p};
|
package/dist/chunk-KJQXUZ4Y.js
DELETED
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
function l(r){let t=[],e=[],n="",i=!1;for(let s=0;s<r.length;s+=1){let o=r[s],c=r[s+1];if(i){o==='"'&&c==='"'?(n+='"',s+=1):o==='"'?i=!1:n+=o;continue}o==='"'?i=!0:o===","?(e.push(n),n=""):o===`
|
|
2
|
-
`?(e.push(n),t.push(e),e=[],n=""):o!=="\r"&&(n+=o)}return(n.length>0||e.length>0)&&(e.push(n),t.push(e)),t}function u(r){let t=l(r).filter(n=>n.some(i=>i.trim()!=="")),e=t[0]?.map(n=>n.trim())??[];return t.slice(1).map(n=>{let i={};for(let s=0;s<e.length;s+=1)i[e[s]]=n[s]??"";return i})}function f(r){if(r==null)return"";let t=String(r);return/[",\n\r]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function g(r,t){return[r.join(","),...t.map(e=>r.map(n=>f(e[n])).join(","))].join(`
|
|
3
|
-
`)+`
|
|
4
|
-
`}export{l as a,u as b,g as c};
|
package/dist/chunk-KXJCGV3H.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{e as i,f as I,h as m,j as o}from"./chunk-SEZETSB3.js";import{i as A}from"./chunk-SDQW7IJN.js";var s="site-extracts/",y=10080*60*1e3,b=900*1e3;function x(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function _(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function r(){return{prefix:s,artifactTtlMs:y,downloadTtlMs:b,token:x()}}function l(t){return A(t,s)}async function v(t){let e=await i({policy:r(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function R(t){let e=await I({policy:r(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function D(t){let e=await i({policy:r(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function S(t){return i({policy:r(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function C(t){return m({policy:r(),artifactId:t.artifactId,ownerId:t.ownerId})}async function P(t){return o({policy:r(),artifactId:t,maxBytes:50*1024*1024})}async function B(t){return l(t.artifactId)!==t.ownerId?null:P(t.artifactId)}async function k(t){return l(t.artifactId)!==t.ownerId?null:o({policy:r(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function h(t){return l(t.artifactId)!==t.ownerId?null:o({policy:r(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function U(t={}){let e=t.now??new Date,a=x();if(!a)return{deleted:0,store:_()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let E=e.getTime()-y,{list:T,del:w}=await import("@vercel/blob"),f,p=0;for(let u=0;u<20;u+=1){let n=await T({prefix:s,token:a,limit:1e3,cursor:f}),c=n.blobs.filter(d=>new Date(d.uploadedAt).getTime()<=E);if(c.length>0&&(await w(c.map(d=>d.pathname),{token:a}),p+=c.length),!n.hasMore||!n.cursor)break;f=n.cursor}return{deleted:p,store:"private-vercel-blob"}}export{s as a,y as b,b as c,r as d,l as e,v as f,R as g,D as h,S as i,C as j,P as k,B as l,k as m,h as n,U as o};
|
package/dist/chunk-NALXKTUU.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{createHmac as s,timingSafeEqual as u}from"crypto";var a=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function f(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(a())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var i=()=>f();function l(e,t){if(e.length!==t.length)return!1;try{return u(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function p(e){let t=String(e),n=s("sha256",i()).update(t).digest("hex");return`${t}.${n}`}function S(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),o=e.slice(t+1),c=s("sha256",i()).update(n).digest("hex");if(!l(o,c))return null;let r=parseInt(n);return isNaN(r)?null:r}export{f as a,p as b,S as c};
|
package/dist/chunk-NNOX3FYG.js
DELETED
|
@@ -1,198 +0,0 @@
|
|
|
1
|
-
import{a as g}from"./chunk-NALXKTUU.js";import{Pool as B}from"pg";var u=class extends Error{constructor(t,s,n=400){super(s);this.code=t;this.status=n;this.name="AnalyticsRepositoryError"}code;status};var D=null;function K(){let i=process.env.ANALYTICS_DATABASE_URL?.trim();if(!i)throw new u("analytics_database_unavailable","Site Analytics is not configured.",503);return i}function E(){return D||(D=new B({connectionString:K(),max:8})),D}import{randomBytes as _e,randomUUID as p}from"crypto";import{createHash as Z,randomUUID as v}from"crypto";import{createCipheriv as X,createDecipheriv as J,createHmac as x,randomBytes as Y,scryptSync as z,timingSafeEqual as V}from"crypto";var N="xray1";function G(){return process.env.ANALYTICS_RESTRICTED_DATA_SECRET?.trim()||g()}function T(i){return z(G(),`xray-restricted:${i}:v1`,32)}function k(i,e){let t=e.trim().slice(0,80);if(!t)throw new Error("analytics encryption purpose required");let s=Y(12),n=X("aes-256-gcm",T(t),s);n.setAAD(Buffer.from(`${N}:${t}`,"utf8"));let r=Buffer.concat([n.update(i,"utf8"),n.final()]),o=n.getAuthTag();return[N,Buffer.from(t,"utf8").toString("base64url"),s.toString("base64url"),o.toString("base64url"),r.toString("base64url")].join(".")}function U(i,e){try{let[t,s,n,r,o]=i.split(".");if(t!==N||!s||!n||!r||!o)return null;let a=Buffer.from(s,"base64url").toString("utf8");if(a!==e)return null;let d=J("aes-256-gcm",T(a),Buffer.from(n,"base64url"));return d.setAAD(Buffer.from(`${N}:${a}`,"utf8")),d.setAuthTag(Buffer.from(r,"base64url")),Buffer.concat([d.update(Buffer.from(o,"base64url")),d.final()]).toString("utf8")}catch{return null}}function Ne(i,e){return x("sha256",T(`identity:${i}`)).update(e.trim().toLowerCase()).digest("hex")}function ve(i,e,t,s){return x("sha256",s).update(`${i}.${e}.${t}`).digest("hex")}function Ae(i,e){if(!i||i.length!==e.length)return!1;try{return V(Buffer.from(i),Buffer.from(e))}catch{return!1}}var M="enhanced_matching",Q=90,ee=10,f=class extends Error{constructor(t,s,n){super(s);this.code=t;this.status=n;this.name="AnalyticsIdentityProfileError"}code;status};function A(i,e){return i?.trim().replace(/\s+/g," ").slice(0,e)||null}function ie(i){if(!i)return null;let[e,t]=i.split("@");return!e||!t?null:`${e.slice(0,1)}${"*".repeat(Math.min(6,Math.max(2,e.length-1)))}@${t}`}function te(i){if(!i)return null;let e=i.replace(/\D/g,"");return e.length<4?null:`***-***-${e.slice(-4)}`}function O(i){return Z("sha256").update(i).digest("hex")}async function h(i,e){if(!e.signals.length)return;let t=e.signals.map(r=>r.kind),s=e.signals.map(r=>r.valueHmac);if((await i.query(`SELECT 1 FROM analytics_identity_tombstones
|
|
2
|
-
WHERE site_id=$1 AND (kind,value_hmac) IN (
|
|
3
|
-
SELECT * FROM unnest($2::text[],$3::text[])
|
|
4
|
-
) LIMIT 1`,[e.siteId,t,s])).rowCount)throw new f("analytics_identity_deleted","This identity cannot be automatically recreated.",410)}async function $(i,e){let t=A(e.displayName,160),s=A(e.email,320)?.toLowerCase()??null,n=A(e.phone,40);if(!t&&!s&&!n)return!1;let o=(await i.query(`SELECT p.public_ref FROM analytics_people p
|
|
5
|
-
JOIN analytics_identity_namespaces n ON n.id=p.namespace_id AND n.site_id=p.site_id AND n.status='approved'
|
|
6
|
-
WHERE p.id=$1 AND p.site_id=$2`,[e.personId,e.siteId])).rows[0]?.public_ref;if(!o)return!1;let a=[...new Set([o,...e.consentSubjectRefs].filter(Boolean))],l=(await i.query(`SELECT s.latest_receipt_id::text FROM analytics_consent_current_state s
|
|
7
|
-
JOIN analytics_consent_receipts r ON r.id=s.latest_receipt_id AND r.site_id=s.site_id
|
|
8
|
-
WHERE s.site_id=$1 AND s.subject_ref=ANY($2::text[])
|
|
9
|
-
AND s.site_analytics=true AND s.enhanced_matching=true AND s.gpc=false
|
|
10
|
-
AND r.action NOT IN ('withdraw','gpc')
|
|
11
|
-
AND (s.expires_at IS NULL OR s.expires_at > now())
|
|
12
|
-
ORDER BY s.effective_at DESC LIMIT 1`,[e.siteId,a])).rows[0]?.latest_receipt_id;if(!l)return!1;let m=e.verifiedAt&&Number.isFinite(Date.parse(e.verifiedAt))?new Date(e.verifiedAt).toISOString():new Date().toISOString(),_=k(JSON.stringify({schemaVersion:1,displayName:t,email:s,phone:n,evidenceType:e.evidenceType,verifiedAt:m}),"identity-profile");return await i.query(`INSERT INTO analytics_restricted_data(id,site_id,person_id,purpose,subject_ref,payload_ciphertext,expires_at,consent_receipt_id)
|
|
13
|
-
VALUES($1,$2,$3,$4,$5,$6,now()+($7::int * interval '1 day'),$8)
|
|
14
|
-
ON CONFLICT(site_id,purpose,subject_ref) DO UPDATE SET
|
|
15
|
-
person_id=EXCLUDED.person_id,payload_ciphertext=EXCLUDED.payload_ciphertext,
|
|
16
|
-
expires_at=EXCLUDED.expires_at,consent_receipt_id=EXCLUDED.consent_receipt_id,created_at=now()`,[v(),e.siteId,e.personId,M,o,_,Q,l]),!0}async function Re(i,e){let t=e.now??new Date,s=await i.query("SELECT role FROM analytics_site_grants WHERE site_id=$1 AND user_id=$2",[e.siteId,e.actorUserId]);if(!s.rowCount)throw new f("analytics_site_not_found","Site not found.",404);if(!["owner","editor"].includes(s.rows[0].role))throw new f("analytics_forbidden","Editor access is required.",403);let n=await i.query(`SELECT count(*)::text AS count FROM analytics_identity_profile_access_audits
|
|
17
|
-
WHERE site_id=$1 AND actor_user_id=$2 AND created_at > $3::timestamptz - interval '1 minute'`,[e.siteId,e.actorUserId,t.toISOString()]);if(Number(n.rows[0]?.count??0)>=ee)throw await i.query(`INSERT INTO analytics_identity_profile_access_audits(id,site_id,actor_user_id,subject_ref_hmac,purpose,result,created_at)
|
|
18
|
-
VALUES($1,$2,$3,$4,'journey_identity_display','rate_limited',$5)`,[v(),e.siteId,e.actorUserId,O(e.requestedPersonRef),t.toISOString()]),new f("analytics_identity_profile_rate_limited","Identity profile rate limit exceeded.",429);let r=async _=>{throw await i.query(`INSERT INTO analytics_identity_profile_access_audits(id,site_id,actor_user_id,subject_ref_hmac,purpose,result,created_at)
|
|
19
|
-
VALUES($1,$2,$3,$4,'journey_identity_display',$5,$6)`,[v(),e.siteId,e.actorUserId,O(e.requestedPersonRef),_,t.toISOString()]),new f("analytics_identity_profile_not_found","Identity profile not found.",404)};if(!e.personId)return r("not_found");let o=await i.query(`SELECT p.public_ref,r.payload_ciphertext,r.expires_at
|
|
20
|
-
FROM analytics_people p
|
|
21
|
-
JOIN analytics_restricted_data r ON r.person_id=p.id AND r.site_id=p.site_id
|
|
22
|
-
AND r.purpose=$3 AND r.subject_ref=p.public_ref AND r.expires_at>$4
|
|
23
|
-
JOIN analytics_consent_current_state s ON s.site_id=p.site_id AND s.latest_receipt_id=r.consent_receipt_id
|
|
24
|
-
AND s.site_analytics=true AND s.enhanced_matching=true AND s.gpc=false
|
|
25
|
-
AND (s.expires_at IS NULL OR s.expires_at>$4)
|
|
26
|
-
JOIN analytics_consent_receipts c ON c.id=s.latest_receipt_id AND c.site_id=s.site_id
|
|
27
|
-
AND c.action NOT IN ('withdraw','gpc')
|
|
28
|
-
WHERE p.id=$1 AND p.site_id=$2
|
|
29
|
-
AND NOT EXISTS (SELECT 1 FROM analytics_identity_tombstones t WHERE t.site_id=p.site_id AND t.person_id=p.id)
|
|
30
|
-
LIMIT 1`,[e.personId,e.siteId,M,t.toISOString()]);if(!o.rowCount)return r("not_found");let a=U(o.rows[0].payload_ciphertext,"identity-profile");if(!a)return r("unavailable");let d;try{d=JSON.parse(a)}catch{return r("unavailable")}let l=d.evidenceType;if(!["first_party_form","authenticated_user","crm_contact"].includes(String(l)))return r("unavailable");let m={personRef:e.requestedPersonRef,displayName:A(typeof d.displayName=="string"?d.displayName:void 0,160),maskedEmail:ie(typeof d.email=="string"?d.email:null),maskedPhone:te(typeof d.phone=="string"?d.phone:null),evidenceType:l,verifiedAt:typeof d.verifiedAt=="string"&&Number.isFinite(Date.parse(d.verifiedAt))?new Date(d.verifiedAt).toISOString():t.toISOString()};return await i.query(`INSERT INTO analytics_identity_profile_access_audits(id,site_id,person_id,actor_user_id,subject_ref_hmac,purpose,result,created_at)
|
|
31
|
-
VALUES($1,$2,$3,$4,$5,'journey_identity_display','returned_masked',$6)`,[v(),e.siteId,e.personId,e.actorUserId,O(e.requestedPersonRef),t.toISOString()]),m}import{randomUUID as re}from"crypto";import{createHmac as se}from"crypto";var ne=new Set(["fbclid","gclid","gbraid","wbraid","ttclid","rdt_cid","msclkid","fbp","fbc","li_fat_id","snapclid"]);function w(i){let e={};for(let[t,s]of Object.entries(i??{})){if(!ne.has(t)||typeof s!="string")continue;let n=s.trim();n&&(e[t]=n.slice(0,500))}return e}function c(i,e){return se("sha256",g()).update(`${i}:${e.trim().toLowerCase()}`).digest("hex")}function oe(i){return i.consent?.analytics==="granted"&&(i.consent.purposes?.enhanced_matching==="granted"||i.consent.adUserData==="granted")&&i.consent.globalPrivacyControl!==!0}function be(i){return!!(i.identity&&oe(i))}async function Pe(i,e){let t=e.event.visitorId?.trim().slice(0,240),s=e.event.consent;if(!t||!s)return null;let n=Date.parse(e.event.occurredAt),r=Date.now();if(!Number.isFinite(n)||n>r+5*6e4||n<r-1440*6e4)return null;let o=s.purposes,a={siteAnalytics:o?o.site_analytics==="granted":s.analytics==="granted",enhancedMatching:o?o.enhanced_matching==="granted":s.adUserData==="granted",advertisingMeasurement:o?o.advertising_measurement==="granted":s.adPersonalization==="granted"},d=s.globalPrivacyControl===!0;d&&(a.siteAnalytics=!1,a.enhancedMatching=!1,a.advertisingMeasurement=!1);let l=d?"gpc":Object.values(a).every(Boolean)?"accept_all":Object.values(a).every(R=>!R)?"reject_all":"save_choices",m=new Date(n).toISOString(),y=(await i.query(`SELECT s.latest_receipt_id::text,s.site_analytics,s.enhanced_matching,s.advertising_measurement,s.gpc,r.action
|
|
32
|
-
FROM analytics_consent_current_state s JOIN analytics_consent_receipts r ON r.id=s.latest_receipt_id
|
|
33
|
-
WHERE s.site_id=$1 AND s.subject_ref=$2 FOR UPDATE`,[e.siteId,t])).rows[0];if(y&&y.site_analytics===a.siteAnalytics&&y.enhanced_matching===a.enhancedMatching&&y.advertising_measurement===a.advertisingMeasurement&&y.gpc===d&&y.action===l)return y.latest_receipt_id;let I=re();return await i.query(`INSERT INTO analytics_consent_receipts(id,site_id,pixel_id,subject_ref,policy_version,jurisdiction_profile,
|
|
34
|
-
necessary,site_analytics,enhanced_matching,advertising_measurement,gpc,action,source,presented_at,occurred_at,supersedes_id)
|
|
35
|
-
VALUES($1,$2,$3,$4,1,'pixel',true,$5,$6,$7,$8,$9,'gateway',$10,$10,$11)`,[I,e.siteId,e.pixelId,t,a.siteAnalytics,a.enhancedMatching,a.advertisingMeasurement,d,l,m,y?.latest_receipt_id??null]),await i.query(`INSERT INTO analytics_consent_current_state(site_id,subject_ref,latest_receipt_id,policy_version,jurisdiction_profile,
|
|
36
|
-
necessary,site_analytics,enhanced_matching,advertising_measurement,gpc,effective_at)
|
|
37
|
-
VALUES($1,$2,$3,1,'pixel',true,$4,$5,$6,$7,$8)
|
|
38
|
-
ON CONFLICT(site_id,subject_ref) DO UPDATE SET latest_receipt_id=EXCLUDED.latest_receipt_id,
|
|
39
|
-
policy_version=EXCLUDED.policy_version,jurisdiction_profile=EXCLUDED.jurisdiction_profile,
|
|
40
|
-
site_analytics=EXCLUDED.site_analytics,enhanced_matching=EXCLUDED.enhanced_matching,
|
|
41
|
-
advertising_measurement=EXCLUDED.advertising_measurement,gpc=EXCLUDED.gpc,
|
|
42
|
-
effective_at=EXCLUDED.effective_at,updated_at=now()
|
|
43
|
-
WHERE EXCLUDED.effective_at>=analytics_consent_current_state.effective_at`,[e.siteId,t,I,a.siteAnalytics,a.enhancedMatching,a.advertisingMeasurement,d,m]),I}async function xe(i,e){if(!e.subjectRef)return{personId:null,activationEligible:!1};let s=(await i.query(`SELECT p.id::text AS person_id,s.advertising_measurement
|
|
44
|
-
FROM analytics_people p
|
|
45
|
-
JOIN analytics_consent_current_state s ON s.site_id=p.site_id AND s.subject_ref=$2
|
|
46
|
-
JOIN analytics_consent_receipts r ON r.id=s.latest_receipt_id AND r.site_id=s.site_id
|
|
47
|
-
WHERE p.site_id=$1
|
|
48
|
-
AND (($3::uuid IS NOT NULL AND p.id=$3) OR ($4::text IS NOT NULL AND EXISTS (
|
|
49
|
-
SELECT 1 FROM analytics_events e WHERE e.site_id=p.site_id AND e.event_id=$4
|
|
50
|
-
AND e.person_id=p.id AND e.visitor_id=$2
|
|
51
|
-
)))
|
|
52
|
-
AND s.site_analytics=true AND s.gpc=false AND r.action NOT IN ('withdraw','gpc')
|
|
53
|
-
AND (s.expires_at IS NULL OR s.expires_at>now()) LIMIT 1`,[e.siteId,e.subjectRef,e.personId??null,e.eventId??null])).rows[0];return s?{personId:s.person_id,activationEligible:s.advertising_measurement===!0}:{personId:null,activationEligible:!1}}async function C(i,e,t,s){return!t?.trim()||!s?.trim()?!1:!!(await i.query(`SELECT 1 FROM analytics_consent_current_state s
|
|
54
|
-
JOIN analytics_consent_receipts r ON r.id=s.latest_receipt_id AND r.site_id=s.site_id
|
|
55
|
-
WHERE s.site_id=$1 AND s.subject_ref=$2
|
|
56
|
-
AND s.site_analytics=true AND s.enhanced_matching=true AND s.gpc=false
|
|
57
|
-
AND r.action NOT IN ('withdraw','gpc')
|
|
58
|
-
AND (s.expires_at IS NULL OR s.expires_at>now())
|
|
59
|
-
AND EXISTS (SELECT 1 FROM analytics_sessions x
|
|
60
|
-
WHERE x.site_id=s.site_id AND x.visitor_hmac=$3 AND x.session_hmac=$4)
|
|
61
|
-
ORDER BY s.effective_at DESC LIMIT 1`,[e,t,c("visitor_id",t),c("session_id",s)])).rowCount}async function H(i,e,t){return!!(await i.query(`SELECT 1 FROM analytics_consent_current_state s
|
|
62
|
-
JOIN analytics_consent_receipts r ON r.id=s.latest_receipt_id AND r.site_id=s.site_id
|
|
63
|
-
WHERE s.site_id=$1 AND s.subject_ref=$2
|
|
64
|
-
AND s.site_analytics=true AND s.enhanced_matching=true AND s.gpc=false
|
|
65
|
-
AND r.action NOT IN ('withdraw','gpc')
|
|
66
|
-
AND (s.expires_at IS NULL OR s.expires_at>now()) LIMIT 1`,[e,t])).rowCount}import{createHash as ae}from"crypto";function He(i,e){let t=i?.trim().toUpperCase();return t&&/^[A-Z0-9-]+$/.test(t)?t.slice(0,e):null}function q(i,e,t){return ae("sha256").update(JSON.stringify({scope:i,siteId:e,filters:t})).digest("base64url").slice(0,18)}function de(i,e){if(!i)return 0;try{let t=JSON.parse(Buffer.from(i,"base64url").toString("utf8"));if(t.v!==1||t.f!==e||!Number.isInteger(t.o)||Number(t.o)<0)throw new Error("invalid");return Number(t.o)}catch{throw new u("invalid_cursor","The pagination cursor is invalid for these filters.")}}function F(i,e,t){if(!e)return{items:i,pageInfo:{hasNextPage:!1,nextCursor:null}};let s=Math.min(100,Math.max(1,Math.trunc(e?.limit??25))),n=de(e?.cursor,t),r=i.slice(n,n+s),o=n+r.length<i.length;return{items:r,pageInfo:{hasNextPage:o,nextCursor:o?Buffer.from(JSON.stringify({v:1,f:t,o:n+r.length})).toString("base64url"):null}}}async function S(i,e,t){let n=(await i.query("SELECT role FROM analytics_site_grants WHERE site_id = $1 AND user_id = $2",[e,t])).rows[0]?.role;if(!n)throw new u("analytics_site_not_found","Site not found.",404);return n}async function ce(i,e,t){let s=await S(i,e,t);if(s==="viewer")throw new u("analytics_forbidden","Editor access is required.",403);return s}async function We(i,e,t){return S(i,e,t)}async function je(i,e,t){return ce(i,e,t)}async function Qe(i){for(;;){let t=await i.query(`SELECT id, visitor_id, session_id
|
|
67
|
-
FROM analytics_events
|
|
68
|
-
WHERE (visitor_id IS NOT NULL AND visitor_hmac IS NULL)
|
|
69
|
-
OR (session_id IS NOT NULL AND session_hmac IS NULL)
|
|
70
|
-
ORDER BY id
|
|
71
|
-
LIMIT $1`,[500]);if(!t.rowCount)break;await i.query(`UPDATE analytics_events e
|
|
72
|
-
SET visitor_hmac = COALESCE(e.visitor_hmac, patch.visitor_hmac),
|
|
73
|
-
session_hmac = COALESCE(e.session_hmac, patch.session_hmac)
|
|
74
|
-
FROM unnest($1::uuid[], $2::text[], $3::text[]) AS patch(id, visitor_hmac, session_hmac)
|
|
75
|
-
WHERE e.id = patch.id`,[t.rows.map(s=>s.id),t.rows.map(s=>s.visitor_id?c("visitor_id",s.visitor_id):null),t.rows.map(s=>s.session_id?c("session_id",s.session_id):null)])}for(;;){let t=await i.query(`SELECT id, session_id
|
|
76
|
-
FROM analytics_conversions
|
|
77
|
-
WHERE session_id IS NOT NULL AND session_hmac IS NULL
|
|
78
|
-
ORDER BY id
|
|
79
|
-
LIMIT $1`,[500]);if(!t.rowCount)break;await i.query(`UPDATE analytics_conversions c
|
|
80
|
-
SET session_hmac = patch.session_hmac
|
|
81
|
-
FROM unnest($1::uuid[], $2::text[]) AS patch(id, session_hmac)
|
|
82
|
-
WHERE c.id = patch.id AND c.session_hmac IS NULL`,[t.rows.map(s=>s.id),t.rows.map(s=>c("session_id",s.session_id))])}}function W(){return`xrp_${_e(18).toString("base64url")}`}function ei(i,e){let t=c("crm_entity_id",`${i}:${e}`).slice(0,32).split("");return t[12]="5",t[16]=["8","9","a","b"][Number.parseInt(t[16],16)%4],`${t.slice(0,8).join("")}-${t.slice(8,12).join("")}-${t.slice(12,16).join("")}-${t.slice(16,20).join("")}-${t.slice(20).join("")}`}async function j(i,e,t){let s=await i.query("SELECT id FROM analytics_identity_namespaces WHERE site_id=$1 AND status='approved' ORDER BY version DESC LIMIT 1",[e]);s.rows[0]||(await i.query(`INSERT INTO analytics_identity_namespaces(
|
|
83
|
-
id,site_id,name,status,approved_hosts,approved_sources,version,approved_by_user_id,approved_at
|
|
84
|
-
)
|
|
85
|
-
SELECT $1,s.id,'X-Ray primary','approved',
|
|
86
|
-
COALESCE((SELECT jsonb_agg(d.hostname ORDER BY d.hostname)
|
|
87
|
-
FROM analytics_pixels p
|
|
88
|
-
JOIN analytics_pixel_domains d ON d.pixel_id=p.id AND d.state='approved'
|
|
89
|
-
WHERE p.site_id=s.id),'[]'::jsonb),
|
|
90
|
-
$3::jsonb,1,s.owner_user_id,now()
|
|
91
|
-
FROM analytics_sites s WHERE s.id=$2
|
|
92
|
-
ON CONFLICT(site_id,name,version) DO NOTHING`,[p(),e,JSON.stringify([t])]),s=await i.query("SELECT id FROM analytics_identity_namespaces WHERE site_id=$1 AND status='approved' ORDER BY version DESC LIMIT 1",[e]));let n=s.rows[0]?.id;if(!n)throw new u("identity_namespace_unavailable","The Site identity graph could not be initialized.",409);return await i.query(`UPDATE analytics_identity_namespaces
|
|
93
|
-
SET approved_sources=(SELECT jsonb_agg(value ORDER BY value)
|
|
94
|
-
FROM (SELECT DISTINCT value
|
|
95
|
-
FROM jsonb_array_elements_text(approved_sources || $3::jsonb)) merged_sources),
|
|
96
|
-
approved_hosts=(SELECT COALESCE(jsonb_agg(hostname ORDER BY hostname),'[]'::jsonb)
|
|
97
|
-
FROM (SELECT DISTINCT hostname
|
|
98
|
-
FROM (SELECT jsonb_array_elements_text(approved_hosts) AS hostname
|
|
99
|
-
UNION ALL
|
|
100
|
-
SELECT d.hostname
|
|
101
|
-
FROM analytics_pixels p
|
|
102
|
-
JOIN analytics_pixel_domains d ON d.pixel_id=p.id AND d.state='approved'
|
|
103
|
-
WHERE p.site_id=$2) hosts) distinct_hosts),
|
|
104
|
-
updated_at=now()
|
|
105
|
-
WHERE id=$1 AND site_id=$2`,[n,e,JSON.stringify([t])]),n}async function le(i,e){if(!e.visitorId&&!e.sessionId)return{events:0,sessions:0,touches:0,conversions:0};let t=e.visitorId?c("visitor_id",e.visitorId):null,s=e.sessionId?c("session_id",e.sessionId):null,n=await i.query(`UPDATE analytics_events SET person_id = $1 WHERE site_id = $2 AND person_id IS NULL
|
|
106
|
-
AND (($3::text IS NOT NULL AND visitor_id = $3) OR ($4::text IS NOT NULL AND session_id = $4))`,[e.personId,e.siteId,e.visitorId??null,e.sessionId??null]),r=await i.query(`UPDATE analytics_conversions SET person_id = $1 WHERE site_id = $2 AND person_id IS NULL
|
|
107
|
-
AND $3::text IS NOT NULL AND session_id = $3`,[e.personId,e.siteId,e.sessionId??null]),o=await i.query(`UPDATE analytics_sessions SET person_id = $1
|
|
108
|
-
WHERE site_id = $2 AND person_id IS NULL
|
|
109
|
-
AND (($3::text IS NOT NULL AND visitor_hmac = $3)
|
|
110
|
-
OR ($4::text IS NOT NULL AND session_hmac = $4))`,[e.personId,e.siteId,t,s]),a=await i.query(`UPDATE analytics_attribution_touches SET person_id = $1
|
|
111
|
-
WHERE site_id = $2 AND person_id IS NULL
|
|
112
|
-
AND (($3::text IS NOT NULL AND visitor_hmac = $3)
|
|
113
|
-
OR ($4::text IS NOT NULL AND session_id IN (
|
|
114
|
-
SELECT id FROM analytics_sessions
|
|
115
|
-
WHERE site_id = $2 AND session_hmac = $4
|
|
116
|
-
)))`,[e.personId,e.siteId,t,s]);return{events:n.rowCount??0,sessions:o.rowCount??0,touches:a.rowCount??0,conversions:r.rowCount??0}}async function L(i,e){let s=(await i.query(`SELECT id FROM analytics_people
|
|
117
|
-
WHERE site_id=$1 AND (namespace_id=$2 OR namespace_id IS NULL)
|
|
118
|
-
AND (($3::uuid IS NOT NULL AND crm_entity_id=$3::uuid)
|
|
119
|
-
OR ($3::uuid IS NULL AND $4::text IS NOT NULL AND crm_person_ref=$4))
|
|
120
|
-
ORDER BY first_seen_at,id LIMIT 1 FOR UPDATE`,[e.siteId,e.namespaceId,e.crmEntityId??null,e.crmPersonRef??null])).rows[0]?.id;if(!s&&!e.crmEntityId&&!e.crmPersonRef&&e.signals.length&&(s=(await i.query(`SELECT assignment.person_id
|
|
121
|
-
FROM analytics_identity_nodes node
|
|
122
|
-
JOIN analytics_identity_assignments assignment
|
|
123
|
-
ON assignment.identity_node_id=node.id AND assignment.site_id=node.site_id
|
|
124
|
-
WHERE node.site_id=$1 AND EXISTS (
|
|
125
|
-
SELECT 1 FROM jsonb_to_recordset($2::jsonb) signal(kind text,value_hmac text)
|
|
126
|
-
WHERE signal.kind=node.kind AND signal.value_hmac=node.value_hmac
|
|
127
|
-
)
|
|
128
|
-
ORDER BY assignment.assigned_at,assignment.person_id LIMIT 1`,[e.siteId,JSON.stringify(e.signals.map(r=>({kind:r.kind,value_hmac:r.valueHmac})))])).rows[0]?.person_id),s)await i.query(`UPDATE analytics_people SET last_seen_at=now(),
|
|
129
|
-
namespace_id=COALESCE(namespace_id,$2),
|
|
130
|
-
crm_entity_id=COALESCE(crm_entity_id,$3::uuid),
|
|
131
|
-
public_ref=COALESCE(public_ref,$4)
|
|
132
|
-
WHERE id=$1 AND site_id=$5`,[s,e.namespaceId,e.crmEntityId??null,W(),e.siteId]);else{let n=e.crmEntityId??null,r=e.crmPersonRef??(n?`XRay::entity::${n}`:`XRay::anonymous::${p()}`);s=(await i.query(`INSERT INTO analytics_people(id,site_id,namespace_id,crm_person_ref,crm_entity_id,public_ref)
|
|
133
|
-
VALUES($1,$2,$3,$4,$5::uuid,$6)
|
|
134
|
-
ON CONFLICT(site_id,crm_person_ref) DO UPDATE SET last_seen_at=now(),
|
|
135
|
-
namespace_id=COALESCE(analytics_people.namespace_id,EXCLUDED.namespace_id),
|
|
136
|
-
crm_entity_id=COALESCE(analytics_people.crm_entity_id,EXCLUDED.crm_entity_id),
|
|
137
|
-
public_ref=COALESCE(analytics_people.public_ref,EXCLUDED.public_ref)
|
|
138
|
-
RETURNING id`,[p(),e.siteId,e.namespaceId,r,n,W()])).rows[0].id}return{personId:s,mergedPersonIds:[]}}async function b(i,e){let s=(await i.query(`SELECT person_id,crm_entity_id::text FROM analytics_identity_assignments
|
|
139
|
-
WHERE identity_node_id=$1 AND site_id=$2 FOR UPDATE`,[e.identityNodeId,e.siteId])).rows[0];if(!s){await i.query(`INSERT INTO analytics_identity_assignments(
|
|
140
|
-
identity_node_id,site_id,person_id,crm_entity_id,evidence_kind
|
|
141
|
-
) VALUES($1,$2,$3,$4::uuid,$5)
|
|
142
|
-
ON CONFLICT(identity_node_id) DO NOTHING`,[e.identityNodeId,e.siteId,e.personId,e.crmEntityId??null,e.evidenceKind]);let n=await i.query("SELECT person_id FROM analytics_identity_assignments WHERE identity_node_id=$1 AND site_id=$2",[e.identityNodeId,e.siteId]);if(n.rows[0]?.person_id===e.personId)return"assigned";if(!n.rows[0])throw new u("identity_assignment_unavailable","Identity assignment could not be recorded.",409);s={person_id:n.rows[0].person_id,crm_entity_id:null}}return s.person_id===e.personId?(await i.query(`UPDATE analytics_identity_assignments SET updated_at=now(),
|
|
143
|
-
crm_entity_id=COALESCE(crm_entity_id,$3::uuid),evidence_kind=$4
|
|
144
|
-
WHERE identity_node_id=$1 AND site_id=$2`,[e.identityNodeId,e.siteId,e.crmEntityId??null,e.evidenceKind]),"existing"):(await i.query(`INSERT INTO analytics_identity_assignment_conflicts(
|
|
145
|
-
id,site_id,identity_node_id,active_person_id,attempted_person_id,
|
|
146
|
-
active_crm_entity_id,attempted_crm_entity_id,evidence_kind
|
|
147
|
-
) VALUES($1,$2,$3,$4,$5,$6::uuid,$7::uuid,$8)
|
|
148
|
-
ON CONFLICT(identity_node_id,active_person_id,attempted_person_id)
|
|
149
|
-
WHERE status='pending' DO UPDATE SET observed_at=now(),evidence_kind=EXCLUDED.evidence_kind`,[p(),e.siteId,e.identityNodeId,s.person_id,e.personId,s.crm_entity_id,e.crmEntityId??null,e.evidenceKind]),"conflict")}async function me(i,e){let s=(await i.query(`WITH person_nodes AS (
|
|
150
|
-
SELECT DISTINCT identity_node_id
|
|
151
|
-
FROM analytics_identity_edges
|
|
152
|
-
WHERE site_id=$1 AND namespace_id=$2 AND person_id=$3
|
|
153
|
-
)
|
|
154
|
-
UPDATE analytics_candidate_associations a
|
|
155
|
-
SET status='promoted',promoted_person_id=$3,promoted_at=$4::timestamptz,updated_at=now()
|
|
156
|
-
WHERE a.site_id=$1 AND a.namespace_id=$2
|
|
157
|
-
AND (a.left_identity_node_id IN (SELECT identity_node_id FROM person_nodes)
|
|
158
|
-
OR a.right_identity_node_id IN (SELECT identity_node_id FROM person_nodes))
|
|
159
|
-
AND (a.status IN ('active','expired') OR (a.status='promoted' AND a.promoted_person_id=$3))
|
|
160
|
-
RETURNING a.id`,[e.siteId,e.namespaceId,e.personId,e.occurredAt])).rows.map(r=>r.id);if(!s.length)return{associationIds:[],touches:0};for(let r of s)await i.query(`INSERT INTO analytics_candidate_promotions(
|
|
161
|
-
id,site_id,namespace_id,association_id,person_id,evidence_kind,
|
|
162
|
-
evidence_ref_hmac,promoted_by,promoted_at
|
|
163
|
-
) VALUES($1,$2,$3,$4,$5,$6,$7,'system',$8::timestamptz)
|
|
164
|
-
ON CONFLICT(association_id,person_id) DO NOTHING`,[p(),e.siteId,e.namespaceId,r,e.personId,e.candidateEvidenceKind,e.evidenceRefHmac,e.occurredAt]);let n=await i.query(`UPDATE analytics_attribution_touches touch SET person_id=$3
|
|
165
|
-
FROM analytics_candidate_attribution_touches candidate_touch
|
|
166
|
-
WHERE candidate_touch.site_id=$1 AND candidate_touch.namespace_id=$2
|
|
167
|
-
AND candidate_touch.association_id=ANY($4::uuid[])
|
|
168
|
-
AND candidate_touch.touch_id=touch.id AND touch.site_id=$1 AND touch.person_id IS NULL`,[e.siteId,e.namespaceId,e.personId,s]);return{associationIds:s,touches:n.rowCount??0}}async function ye(i,e){let t=c("promotion_evidence",`${e.evidenceKind}:${e.evidenceRef}`),s=c("promotion_person",e.personId),n=c("promotion_receipt",`${e.siteId}:${e.personId}:${e.evidenceKind}:${e.evidenceRef}`);await i.query(`INSERT INTO analytics_identity_promotion_receipts(
|
|
169
|
-
id,site_id,namespace_id,canonical_person_id,canonical_person_ref_hmac,
|
|
170
|
-
evidence_kind,evidence_ref_hmac,prior_person_ids,prior_identity_node_ids,
|
|
171
|
-
candidate_association_ids,affected_history_counts,schema_version,idempotency_key,promoted_at
|
|
172
|
-
) VALUES($1,$2,$3,$4,$5,$6,$7,$8::uuid[],$9::uuid[],$10::uuid[],$11::jsonb,
|
|
173
|
-
'xray.identity-promotion.v1',$12,$13::timestamptz)
|
|
174
|
-
ON CONFLICT(site_id,idempotency_key) DO NOTHING`,[p(),e.siteId,e.namespaceId,e.personId,s,e.evidenceKind,t,e.priorPersonIds,e.priorIdentityNodeIds,e.candidateAssociationIds,JSON.stringify({...e.history,candidateTouches:e.candidateTouches}),n,e.occurredAt])}async function P(i,e){let t=e.occurredAt??new Date().toISOString(),s=await le(i,e),n=c("promotion_evidence",`${e.evidenceKind}:${e.evidenceRef}`),r=await me(i,{siteId:e.siteId,namespaceId:e.namespaceId,personId:e.personId,candidateEvidenceKind:e.candidateEvidenceKind,evidenceRefHmac:n,occurredAt:t});await ye(i,{...e,history:s,candidateAssociationIds:r.associationIds,candidateTouches:r.touches,occurredAt:t})}async function ii(i,e){let t=e.customerId||e.orderId||e.email||e.phone;if(!t||!await C(i,e.siteId,e.visitorId,e.sessionId))return null;let s=await j(i,e.siteId,"pixel"),n=[...e.email?[{kind:"email",valueHmac:c("email",e.email)}]:[],...e.phone?[{kind:"phone",valueHmac:c("phone",e.phone)}]:[],...e.customerId?[{kind:"customer_id",valueHmac:c("customer_id",e.customerId)}]:[],...e.orderId?[{kind:"customer_id",valueHmac:c("customer_id",`order:${e.orderId}`)}]:[]];await h(i,{siteId:e.siteId,signals:n});let r=`XRay::${c("person_ref",t).slice(0,24)}`,o=[...n,...e.visitorId?[{kind:"visitor_id",valueHmac:c("visitor_id",e.visitorId)}]:[],...e.sessionId?[{kind:"session_id",valueHmac:c("session_id",e.sessionId)}]:[]],a=await L(i,{siteId:e.siteId,namespaceId:s,crmPersonRef:r,signals:o}),d=a.personId,l=[];e.visitorId&&l.push({kind:"visitor_id",value:e.visitorId,evidence:"pixel_identify",confidence:.98}),e.sessionId&&l.push({kind:"session_id",value:e.sessionId,evidence:"pixel_identify",confidence:.98}),e.email&&l.push({kind:"email",value:e.email,evidence:"pixel_identify",confidence:1}),e.phone&&l.push({kind:"phone",value:e.phone,evidence:"pixel_identify",confidence:.99}),e.customerId&&l.push({kind:"customer_id",value:e.customerId,evidence:"customer_id",confidence:1}),e.orderId&&l.push({kind:"customer_id",value:`order:${e.orderId}`,evidence:"order_id",confidence:1});for(let[y,I]of Object.entries(w(e.clickIds)))l.push({kind:"click_id",value:`${y}:${I}`,evidence:y,confidence:.9});let m=[],_=!1;for(let y of l){let I=await i.query(`INSERT INTO analytics_identity_nodes(id, site_id, namespace_id, kind, value_hmac) VALUES ($1,$2,$3,$4,$5)
|
|
175
|
-
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now(),
|
|
176
|
-
namespace_id=COALESCE(analytics_identity_nodes.namespace_id,EXCLUDED.namespace_id) RETURNING id`,[p(),e.siteId,s,y.kind,c(y.kind,y.value)]);m.push(I.rows[0].id),await i.query(`INSERT INTO analytics_identity_edges(id, site_id, namespace_id, person_id, identity_node_id, evidence_kind, confidence)
|
|
177
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
178
|
-
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE
|
|
179
|
-
SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence),
|
|
180
|
-
namespace_id=COALESCE(analytics_identity_edges.namespace_id,EXCLUDED.namespace_id)`,[p(),e.siteId,s,d,I.rows[0].id,y.evidence,y.confidence]);let R=await b(i,{siteId:e.siteId,personId:d,identityNodeId:I.rows[0].id,evidenceKind:y.evidence});_||=R==="conflict"}return _||await P(i,{siteId:e.siteId,namespaceId:s,personId:d,evidenceKind:e.customerId?"authenticated_login":"browser_identify",candidateEvidenceKind:e.email?"verified_email":e.phone?"verified_phone":"authenticated_user",evidenceRef:t,priorPersonIds:a.mergedPersonIds,priorIdentityNodeIds:m,visitorId:e.visitorId,sessionId:e.sessionId,occurredAt:e.occurredAt}),e.customerId&&!_&&await $(i,{siteId:e.siteId,personId:d,consentSubjectRefs:[e.visitorId??"",e.sessionId??""],evidenceType:"authenticated_user",email:e.email,phone:e.phone}),d}async function ti(i){let e=await E().connect();try{if(await e.query("BEGIN"),!await C(e,i.siteId,i.visitorId,i.sessionId))return await e.query("COMMIT"),{personId:null,linkedSignals:0,conflicts:0};let t=await j(e,i.siteId,"form"),s=[...i.email?[{kind:"email",valueHmac:c("email",i.email)}]:[],...i.phone?[{kind:"phone",valueHmac:c("phone",i.phone)}]:[]];await h(e,{siteId:i.siteId,signals:s});let n=[...s,...i.visitorId?[{kind:"visitor_id",valueHmac:c("visitor_id",i.visitorId)}]:[],...i.sessionId?[{kind:"session_id",valueHmac:c("session_id",i.sessionId)}]:[]],r=await L(e,{siteId:i.siteId,namespaceId:t,crmPersonRef:i.crmPersonRef,crmEntityId:i.crmEntityId,signals:n}),o=r.personId,a=[];i.visitorId&&a.push({kind:"visitor_id",value:i.visitorId,evidence:"first_party_form",confidence:.98}),i.sessionId&&a.push({kind:"session_id",value:i.sessionId,evidence:"first_party_form",confidence:.95}),i.email&&a.push({kind:"email",value:i.email,evidence:"submitted_email",confidence:1}),i.phone&&a.push({kind:"phone",value:i.phone,evidence:"submitted_phone",confidence:.99});for(let[m,_]of Object.entries(w(i.clickIds)))a.push({kind:"click_id",value:`${m}:${_}`,evidence:m,confidence:.9});let d=[],l=0;for(let m of a){let _=await e.query(`INSERT INTO analytics_identity_nodes(id, site_id, namespace_id, kind, value_hmac) VALUES ($1,$2,$3,$4,$5)
|
|
181
|
-
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now(),
|
|
182
|
-
namespace_id=COALESCE(analytics_identity_nodes.namespace_id,EXCLUDED.namespace_id) RETURNING id`,[p(),i.siteId,t,m.kind,c(m.kind,m.value)]);d.push(_.rows[0].id),await e.query(`INSERT INTO analytics_identity_edges(id, site_id, namespace_id, person_id, identity_node_id, evidence_kind, confidence)
|
|
183
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
184
|
-
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at = now(),
|
|
185
|
-
confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence),
|
|
186
|
-
namespace_id=COALESCE(analytics_identity_edges.namespace_id,EXCLUDED.namespace_id)`,[p(),i.siteId,t,o,_.rows[0].id,m.evidence,m.confidence]),await b(e,{siteId:i.siteId,personId:o,crmEntityId:i.crmEntityId,identityNodeId:_.rows[0].id,evidenceKind:m.evidence})==="conflict"&&(l+=1)}return i.submissionId&&await e.query("UPDATE analytics_form_submissions SET person_id = $1 WHERE id = $2 AND site_id = $3",[o,i.submissionId,i.siteId]),l===0&&await P(e,{siteId:i.siteId,namespaceId:t,personId:o,evidenceKind:"form_fill",candidateEvidenceKind:i.email?"verified_email":i.phone?"verified_phone":"authenticated_user",evidenceRef:i.submissionId??i.crmEntityId??i.crmPersonRef??o,priorPersonIds:r.mergedPersonIds,priorIdentityNodeIds:d,visitorId:i.visitorId,sessionId:i.sessionId}),l===0&&await $(e,{siteId:i.siteId,personId:o,consentSubjectRefs:[i.visitorId??""],evidenceType:i.evidenceType??"first_party_form",displayName:i.displayName,email:i.email,phone:i.phone}),await e.query("COMMIT"),{personId:o,linkedSignals:a.length,conflicts:l}}catch(t){throw await e.query("ROLLBACK"),t}finally{e.release()}}async function si(i){let e=await E().connect();try{await e.query("BEGIN");let s=(await e.query(`SELECT p.id,p.public_ref,p.namespace_id FROM analytics_people p
|
|
187
|
-
JOIN analytics_identity_namespaces n ON n.id=p.namespace_id AND n.site_id=p.site_id AND n.status='approved'
|
|
188
|
-
WHERE p.site_id=$1 AND p.crm_person_ref=$2 LIMIT 1 FOR UPDATE OF p`,[i.siteId,i.crmPersonRef])).rows[0];if(!s||!await H(e,i.siteId,s.public_ref))return await e.query("COMMIT"),{personId:null,linkedSignals:0,conflicts:0};let n=[...i.email?[{kind:"email",value:i.email,confidence:1}]:[],...i.phone?[{kind:"phone",value:i.phone,confidence:.99}]:[]];await h(e,{siteId:i.siteId,signals:n.map(_=>({kind:_.kind,valueHmac:c(_.kind,_.value)}))});let r=await L(e,{siteId:i.siteId,namespaceId:s.namespace_id,crmPersonRef:i.crmPersonRef,crmEntityId:i.crmEntityId,signals:n.map(_=>({kind:_.kind,valueHmac:c(_.kind,_.value)}))}),o=r.personId,d=(await e.query("SELECT public_ref FROM analytics_people WHERE id=$1 AND site_id=$2",[o,i.siteId])).rows[0]?.public_ref??s.public_ref,l=[],m=0;for(let _ of n){let y=await e.query(`INSERT INTO analytics_identity_nodes(id,site_id,namespace_id,kind,value_hmac) VALUES($1,$2,$3,$4,$5)
|
|
189
|
-
ON CONFLICT(site_id,kind,value_hmac) DO UPDATE SET last_seen_at=now(),
|
|
190
|
-
namespace_id=COALESCE(analytics_identity_nodes.namespace_id,EXCLUDED.namespace_id) RETURNING id`,[p(),i.siteId,s.namespace_id,_.kind,c(_.kind,_.value)]);l.push(y.rows[0].id),await e.query(`INSERT INTO analytics_identity_edges(id,site_id,namespace_id,person_id,identity_node_id,evidence_kind,confidence)
|
|
191
|
-
VALUES($1,$2,$3,$4,$5,'crm_contact',$6)
|
|
192
|
-
ON CONFLICT(person_id,identity_node_id,evidence_kind) DO UPDATE SET last_seen_at=now(),
|
|
193
|
-
confidence=greatest(analytics_identity_edges.confidence,EXCLUDED.confidence)`,[p(),i.siteId,s.namespace_id,o,y.rows[0].id,_.confidence]),await b(e,{siteId:i.siteId,personId:o,crmEntityId:i.crmEntityId,identityNodeId:y.rows[0].id,evidenceKind:"crm_contact"})==="conflict"&&(m+=1)}return m===0&&await P(e,{siteId:i.siteId,namespaceId:s.namespace_id,personId:o,evidenceKind:"crm_identity",candidateEvidenceKind:"crm_contact",evidenceRef:i.crmPersonRef,priorPersonIds:r.mergedPersonIds,priorIdentityNodeIds:l}),m===0&&await $(e,{siteId:i.siteId,personId:o,consentSubjectRefs:[d],evidenceType:"crm_contact",displayName:i.displayName,email:i.email,phone:i.phone}),await e.query("COMMIT"),{personId:o,linkedSignals:n.length,conflicts:m}}catch(t){throw await e.query("ROLLBACK"),t}finally{e.release()}}async function ni(i,e,t){let s=E();await S(s,i,e);let n=await s.query(`SELECT p.id, p.first_seen_at, p.last_seen_at, true AS crm_linked,
|
|
194
|
-
(SELECT count(*)::int FROM analytics_identity_edges e WHERE e.person_id = p.id) AS signal_count,
|
|
195
|
-
(SELECT count(*)::int FROM analytics_form_submissions f WHERE f.person_id = p.id) AS form_submission_count,
|
|
196
|
-
(SELECT count(*)::int FROM analytics_conversions c WHERE c.person_id = p.id) AS conversion_count,
|
|
197
|
-
(SELECT COALESCE(sum(c.value_minor), 0)::bigint FROM analytics_conversions c WHERE c.person_id = p.id) AS revenue_minor
|
|
198
|
-
FROM analytics_people p WHERE p.site_id = $1 ORDER BY p.last_seen_at DESC, p.id DESC LIMIT 1000`,[i]),r=F(n.rows,t,q("people",i,{start:"all",end:"all"}));return{people:r.items,pageInfo:r.pageInfo}}export{u as a,E as b,k as c,U as d,Ne as e,ve as f,Ae as g,w as h,c as i,f as j,Re as k,oe as l,be as m,Pe as n,xe as o,He as p,q,F as r,S as s,ce as t,We as u,je as v,Qe as w,W as x,ei as y,le as z,b as A,ii as B,ti as C,si as D,ni as E};
|