@wenathlan/devthink 2.0.12 → 2.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/accounts.ts +2087 -0
- package/agent.ts +2331 -0
- package/antigravity.ts +2139 -0
- package/apifreeze.ts +4146 -0
- package/auth.ts +642 -0
- package/background.ts +16837 -0
- package/bridge.ts +2080 -0
- package/build.ts +54 -0
- package/bun.ts +26 -0
- package/capture.ts +1614 -0
- package/checksecrets.ts +72 -0
- package/cjs.ts +9 -0
- package/cli.ts +3500 -0
- package/commands.ts +1101 -0
- package/companion.ts +450 -0
- package/compatibility.ts +37 -0
- package/config.ts +3137 -0
- package/constants.ts +943 -0
- package/core.ts +1473 -0
- package/crossbrowser.ts +1359 -0
- package/dashdone.ts +605 -0
- package/data.ts +943 -0
- package/database.ts +133 -0
- package/debug.ts +850 -0
- package/deno.ts +27 -0
- package/devthink.ts +707 -0
- package/dist/accounts.d.ts +409 -0
- package/dist/agent.d.ts +837 -0
- package/dist/antigravity.d.ts +380 -0
- package/dist/apifreeze.d.ts +113 -0
- package/dist/auth.d.ts +224 -0
- package/{bridge.d.ts → dist/bridge.d.ts} +1 -1
- package/dist/capture.d.ts +595 -0
- package/dist/cjs.d.ts +6 -0
- package/{cli.js → dist/cli.js} +2 -2
- package/dist/commands.d.ts +310 -0
- package/dist/compatibility.d.ts +17 -0
- package/dist/config.d.ts +1853 -0
- package/dist/constants.d.ts +637 -0
- package/dist/core.d.ts +361 -0
- package/dist/data.d.ts +330 -0
- package/dist/database.d.ts +24 -0
- package/dist/debug.d.ts +270 -0
- package/dist/engine.d.ts +74 -0
- package/dist/environments.d.ts +244 -0
- package/dist/evidence.d.ts +277 -0
- package/dist/export.d.ts +311 -0
- package/dist/fingerprint.d.ts +594 -0
- package/dist/flow.d.ts +300 -0
- package/dist/gates.d.ts +103 -0
- package/dist/identity.d.ts +51 -0
- package/dist/ids.d.ts +7 -0
- package/dist/llm.d.ts +652 -0
- package/dist/models.d.ts +1807 -0
- package/dist/net.d.ts +325 -0
- package/dist/neutral.d.ts +41 -0
- package/dist/oauth.d.ts +703 -0
- package/dist/page.d.ts +1166 -0
- package/dist/perf.d.ts +481 -0
- package/dist/plan.d.ts +134 -0
- package/dist/project.d.ts +395 -0
- package/dist/providers.d.ts +37 -0
- package/dist/quota.d.ts +453 -0
- package/dist/recovery.d.ts +621 -0
- package/dist/request.d.ts +1492 -0
- package/dist/run.d.ts +779 -0
- package/dist/runtime.d.ts +212 -0
- package/dist/search.d.ts +375 -0
- package/dist/security.d.ts +534 -0
- package/dist/serve.d.ts +136 -0
- package/dist/session.d.ts +296 -0
- package/dist/storage.d.ts +53 -0
- package/dist/streaming.d.ts +1119 -0
- package/dist/swarm.d.ts +328 -0
- package/dist/tools.d.ts +170 -0
- package/dist/types.d.ts +8357 -0
- package/dist/utils.d.ts +82 -0
- package/dist/version.d.ts +7 -0
- package/dist/versionregistry.d.ts +406 -0
- package/dist/views.d.ts +579 -0
- package/dist/workbenchsession.d.ts +69 -0
- package/dist/workflow.d.ts +649 -0
- package/docs/readme.md +92 -0
- package/docs/schemas/antigravity.json +192 -0
- package/engine.ts +1524 -0
- package/environments.ts +1005 -0
- package/evidence.ts +643 -0
- package/export.ts +889 -0
- package/fingerprint.ts +1458 -0
- package/flow.ts +1190 -0
- package/gates.ts +307 -0
- package/gateway.ts +34 -0
- package/hardening.ts +191 -0
- package/headless.ts +369 -0
- package/http.ts +2110 -0
- package/identity.ts +246 -0
- package/ids.ts +26 -0
- package/index.ts +369 -0
- package/llm.ts +2682 -0
- package/mcp.ts +1835 -0
- package/{memory.js → memory.ts} +3834 -2310
- package/models.ts +3805 -0
- package/modes.ts +43 -0
- package/net.ts +1214 -0
- package/neutral.ts +362 -0
- package/node.ts +26 -0
- package/oauth.ts +2216 -0
- package/offscreen.ts +170 -0
- package/pack.ts +956 -0
- package/package.json +175 -82
- package/page.ts +7824 -0
- package/pagebridge.ts +1116 -0
- package/perf.ts +957 -0
- package/plan.ts +672 -0
- package/plugin.ts +76 -0
- package/{policy.js → policy.ts} +5073 -4679
- package/prisma.config.ts +27 -0
- package/progress.ts +1308 -0
- package/project.ts +1291 -0
- package/protocol.ts +4187 -0
- package/providers.ts +246 -0
- package/quota.ts +1887 -0
- package/recovery.ts +2433 -0
- package/request.ts +4482 -0
- package/run.ts +2135 -0
- package/runtime.ts +526 -0
- package/search.ts +1760 -0
- package/security.ts +1673 -0
- package/serve.ts +419 -0
- package/server.ts +5538 -0
- package/session.ts +1162 -0
- package/storage.ts +182 -0
- package/streaming.ts +4101 -0
- package/swarm.ts +884 -0
- package/sync.ts +56 -0
- package/tests/agentcert.test.ts +128 -0
- package/tests/agentfleet.test.ts +214 -0
- package/tests/agentreview.test.ts +241 -0
- package/tests/agentstream.test.ts +335 -0
- package/tests/agentwork.test.ts +356 -0
- package/tests/apifreeze.test.ts +290 -0
- package/tests/apimap.test.ts +112 -0
- package/tests/approvalgate.test.ts +175 -0
- package/tests/artifactmanifest.test.ts +204 -0
- package/tests/attentionfeed.test.ts +163 -0
- package/tests/backgroundruns.test.ts +100 -0
- package/tests/batchscheduling.test.ts +131 -0
- package/tests/blackboard.test.ts +238 -0
- package/tests/bridge.test.ts +1088 -0
- package/tests/browsercoverage.test.ts +134 -0
- package/tests/browserpolyfills.test.ts +124 -0
- package/tests/build.mjs +1260 -0
- package/tests/clientauth.test.ts +223 -0
- package/tests/clitools.test.ts +578 -0
- package/tests/confirmgates.test.ts +190 -0
- package/tests/containerpack.test.ts +143 -0
- package/tests/controlflow.test.ts +2120 -0
- package/tests/coordination.test.ts +345 -0
- package/tests/copyscreen.test.ts +139 -0
- package/tests/core.test.ts +417 -0
- package/tests/costcert.test.ts +95 -0
- package/tests/crossbrowser.test.ts +115 -0
- package/tests/cspaudit.test.ts +79 -0
- package/tests/dashdone.test.ts +615 -0
- package/tests/datagrid.test.ts +137 -0
- package/tests/doccheck.test.ts +107 -0
- package/tests/dombatch.test.ts +109 -0
- package/tests/emulation.test.ts +282 -0
- package/tests/environments.test.ts +259 -0
- package/tests/evidenceviews.test.ts +88 -0
- package/tests/exporttools.test.ts +185 -0
- package/tests/firefoxprep.test.ts +134 -0
- package/tests/flowlibrary.test.ts +242 -0
- package/tests/flowrun.test.ts +303 -0
- package/tests/forensics.test.ts +439 -0
- package/tests/gateway.test.ts +939 -0
- package/tests/headlesslib.test.ts +171 -0
- package/tests/httpstream.test.ts +277 -0
- package/tests/immutablelog.test.ts +157 -0
- package/tests/inboundguard.test.ts +177 -0
- package/tests/lazyload.test.ts +72 -0
- package/tests/librarymodes.test.ts +623 -0
- package/tests/llm.test.ts +696 -0
- package/tests/manifest.test.ts +226 -0
- package/tests/maskinputs.test.ts +150 -0
- package/tests/matrixverify.test.ts +111 -0
- package/tests/mavenpack.test.ts +75 -0
- package/tests/mcpserver.test.ts +704 -0
- package/tests/memory.test.ts +4155 -0
- package/tests/memorycare.test.ts +230 -0
- package/tests/migrateplan.test.ts +349 -0
- package/tests/minimization.test.ts +414 -0
- package/tests/modelroute.test.ts +112 -0
- package/tests/multiagent.test.ts +440 -0
- package/tests/nativebridge.test.ts +807 -0
- package/tests/navigation.test.ts +284 -0
- package/tests/nugetpack.test.ts +83 -0
- package/tests/orchestration.test.ts +449 -0
- package/tests/originpolicy.test.ts +405 -0
- package/tests/outputcompare.test.ts +97 -0
- package/tests/package.test.ts +91 -0
- package/tests/pagebridge.test.ts +5909 -0
- package/tests/pentest.test.ts +75 -0
- package/tests/perfrecords.test.ts +257 -0
- package/tests/permdiff.test.ts +85 -0
- package/tests/phishguard.test.ts +128 -0
- package/tests/pickerviews.test.ts +155 -0
- package/tests/pipelines.test.ts +381 -0
- package/tests/planlint.test.ts +240 -0
- package/tests/planreview.test.ts +383 -0
- package/tests/platformtargets.test.ts +56 -0
- package/tests/policy.test.ts +8284 -0
- package/tests/portability.test.ts +96 -0
- package/tests/progress.test.ts +1036 -0
- package/tests/promptlibrary.test.ts +116 -0
- package/tests/protocol.test.ts +4793 -0
- package/tests/publishingpipeline.test.ts +314 -0
- package/tests/quickactions.test.ts +164 -0
- package/tests/readiness.test.ts +78 -0
- package/tests/recipes.test.ts +163 -0
- package/tests/redactshots.test.ts +216 -0
- package/tests/registernvidiakeys.ts +51 -0
- package/tests/relayserve.test.ts +304 -0
- package/tests/releasenotes.test.ts +276 -0
- package/tests/resilience.test.ts +467 -0
- package/tests/resourceaware.test.ts +187 -0
- package/tests/runbudget.test.ts +125 -0
- package/tests/runhistory.test.ts +299 -0
- package/tests/runreplay.test.ts +113 -0
- package/tests/runresume.test.ts +184 -0
- package/tests/runstate.test.ts +288 -0
- package/tests/runtimeadapters.test.ts +251 -0
- package/tests/safariskeleton.test.ts +115 -0
- package/tests/sandboxframe.test.ts +196 -0
- package/tests/sbom.test.ts +105 -0
- package/tests/secretvault.test.ts +188 -0
- package/tests/selcache.test.ts +111 -0
- package/tests/sessioninterface.test.ts +791 -0
- package/tests/sessions.test.ts +591 -0
- package/tests/siteprefs.test.ts +228 -0
- package/tests/snapshotdelta.test.ts +165 -0
- package/tests/statusviews.test.ts +237 -0
- package/tests/stepmeter.test.ts +200 -0
- package/tests/streamparse.test.ts +163 -0
- package/tests/stylepruning.test.ts +90 -0
- package/tests/surfaces.test.ts +282 -0
- package/tests/sweep.test.ts +78 -0
- package/tests/syncbridge.test.ts +164 -0
- package/tests/taskqueue.test.ts +206 -0
- package/tests/toolcalls.test.ts +336 -0
- package/tests/toolcatalog.test.ts +201 -0
- package/tests/torture-agent.test.ts +706 -0
- package/tests/torture-auth.test.ts +461 -0
- package/tests/torture-commands.test.ts +895 -0
- package/tests/torture-data.test.ts +923 -0
- package/tests/torture-environments.test.ts +285 -0
- package/tests/torture-export.test.ts +263 -0
- package/tests/torture-http.test.ts +1379 -0
- package/tests/torture-llm.test.ts +808 -0
- package/tests/torture-memory.test.ts +1084 -0
- package/tests/torture-plan.test.ts +829 -0
- package/tests/torture-run.test.ts +626 -0
- package/tests/torture-session.test.ts +928 -0
- package/tests/torture-swarm.test.ts +1245 -0
- package/tests/torture-tools.test.ts +964 -0
- package/tests/torture-workflow.test.ts +2354 -0
- package/tests/tourviews.test.ts +52 -0
- package/tests/transparency.test.ts +147 -0
- package/tests/trigger.test.ts +941 -0
- package/tests/v1sunset.test.ts +428 -0
- package/tests/virtlist.test.ts +94 -0
- package/tests/vision.test.ts +540 -0
- package/tests/vsixpack.test.ts +145 -0
- package/tests/webapi.test.ts +523 -0
- package/tests/workflow.test.ts +1785 -0
- package/tests/workfloweditor.test.ts +759 -0
- package/tests/xpipack.test.ts +128 -0
- package/tools.ts +734 -0
- package/types.ts +9908 -0
- package/umd.ts +9 -0
- package/utils.ts +249 -0
- package/version.ts +8 -0
- package/versionregistry.ts +1310 -0
- package/views.ts +2153 -0
- package/vitest.config.ts +12 -0
- package/web/App.tsx +78 -0
- package/web/README.md +47 -0
- package/web/capacitor.config.ts +46 -0
- package/web/const.ts +19 -0
- package/web/control.shell.tsx +66 -0
- package/web/dashboardpage.ts +554 -0
- package/web/db.ts +493 -0
- package/web/design.html +541 -0
- package/web/gatewayview/Gateway.tsx +345 -0
- package/web/gatewayview/config.ts +1042 -0
- package/web/gatewayview/definition.ts +174 -0
- package/web/icons.ts +15 -0
- package/{manifest.json → web/manifest.json} +1 -1
- package/web/optionspage.ts +691 -0
- package/web/popup.ts +1651 -0
- package/web/schema.prisma +176 -0
- package/web/sidepanel.ts +7941 -0
- package/web/sitemanifest.json +13 -0
- package/web/terminal-ui.tsx +622 -0
- package/web/transparencypage.ts +541 -0
- package/web/vite.config.ts +39 -0
- package/web/workspace.ts +23 -0
- package/workbenchmemory.ts +104 -0
- package/workbenchsession.ts +304 -0
- package/workflow.ts +4495 -0
- package/bridge.d.ts.map +0 -1
- package/bridge.js +0 -1877
- package/bun.d.ts.map +0 -1
- package/bun.js +0 -52329
- package/caps/background.json +0 -712
- package/caps/cli.json +0 -360
- package/caps/library.json +0 -3611
- package/caps/mcp.json +0 -126
- package/caps/pagebridge.json +0 -377
- package/caps/popup.json +0 -51
- package/caps/sidepanel.json +0 -522
- package/checksums.txt +0 -142
- package/cli.d.ts.map +0 -1
- package/companion.d.ts.map +0 -1
- package/companion.js +0 -389
- package/crossbrowser.d.ts.map +0 -1
- package/crossbrowser.js +0 -1191
- package/dashdone.d.ts.map +0 -1
- package/dashdone.js +0 -368
- package/deno.d.ts.map +0 -1
- package/deno.js +0 -52325
- package/devthink.umd.js +0 -52346
- package/fixtures/bun.ts +0 -7
- package/fixtures/cjs.js +0 -6
- package/fixtures/deno.ts +0 -6
- package/fixtures/example-org-pagestate.json +0 -22
- package/fixtures/importers/automa-workflow.json +0 -18
- package/fixtures/importers/selenium-side.json +0 -20
- package/fixtures/importers/tabular-plan.csv +0 -7
- package/fixtures/importers/uivision-macro.json +0 -10
- package/fixtures/importers/v1-plan.json +0 -13
- package/fixtures/mcp-client.mjs +0 -14
- package/fixtures/pages/checkout.html +0 -192
- package/fixtures/pages/dashboard.html +0 -110
- package/fixtures/pages/feed.html +0 -113
- package/fixtures/pages/product-grid.html +0 -150
- package/fixtures/plans/form-inventory-plan.json +0 -11
- package/fixtures/plans/release-notes-plan.json +0 -11
- package/fixtures/recipes/agentscompete.json +0 -11
- package/fixtures/recipes/agentsforms.json +0 -11
- package/fixtures/recipes/agentsmonitor.json +0 -11
- package/fixtures/recipes/agentsreview.json +0 -29
- package/fixtures/recipes/agentswarm.json +0 -11
- package/fixtures/recipes/fillcalendar.json +0 -12
- package/fixtures/recipes/fillcheckout.json +0 -13
- package/fixtures/recipes/filldependentdropdowns.json +0 -12
- package/fixtures/recipes/fillfromcsv.json +0 -13
- package/fixtures/recipes/fillloginform.json +0 -12
- package/fixtures/recipes/fillmultistepwizard.json +0 -13
- package/fixtures/recipes/fillregistration.json +0 -12
- package/fixtures/recipes/fillsearchfilters.json +0 -12
- package/fixtures/recipes/filltypeahead.json +0 -11
- package/fixtures/recipes/monitoralerts.json +0 -12
- package/fixtures/recipes/monitoravailability.json +0 -11
- package/fixtures/recipes/monitornetworkquiet.json +0 -12
- package/fixtures/recipes/monitorpagechanges.json +0 -11
- package/fixtures/recipes/monitorpricechange.json +0 -11
- package/fixtures/recipes/monitorschedule.json +0 -12
- package/fixtures/recipes/scrapedynamicprices.json +0 -13
- package/fixtures/recipes/scrapeimagegallery.json +0 -12
- package/fixtures/recipes/scrapeinfinitefeed.json +0 -13
- package/fixtures/recipes/scrapenestedlists.json +0 -12
- package/fixtures/recipes/scrapeproductgrid.json +0 -12
- package/fixtures/recipes/scrapereviews.json +0 -12
- package/fixtures/recipes/scrapesearchresults.json +0 -12
- package/fixtures/recipes/scrapetablepagination.json +0 -12
- package/fixtures/recipes/testaccessibility.json +0 -34
- package/fixtures/recipes/testconsoleerrors.json +0 -12
- package/fixtures/recipes/testdeeplinks.json +0 -12
- package/fixtures/recipes/testformvalidation.json +0 -13
- package/fixtures/recipes/testlayoutshift.json +0 -12
- package/fixtures/recipes/testlinkcheck.json +0 -12
- package/fixtures/recipes/testloadtiming.json +0 -12
- package/fixtures/recipes/testvisualdiff.json +0 -12
- package/fixtures/workflows/changelog-digest-workflow.json +0 -13
- package/fixtures/workflows/form-survey-workflow.json +0 -13
- package/gallery.json +0 -706
- package/hardening.d.ts.map +0 -1
- package/hardening.js +0 -188
- package/headless.d.ts.map +0 -1
- package/headless.js +0 -11522
- package/http.d.ts.map +0 -1
- package/http.js +0 -2165
- package/index.cjs +0 -54682
- package/index.d.ts.map +0 -1
- package/index.js +0 -52310
- package/index.neutral.js +0 -46343
- package/mcp.d.ts.map +0 -1
- package/mcp.js +0 -20515
- package/memory.d.ts.map +0 -1
- package/nativehost.template.json +0 -10
- package/node.cjs +0 -54686
- package/node.d.ts.map +0 -1
- package/pack.d.ts.map +0 -1
- package/pack.js +0 -805
- package/policy.d.ts.map +0 -1
- package/progress.d.ts.map +0 -1
- package/progress.js +0 -645
- package/protocol.d.ts.map +0 -1
- package/protocol.js +0 -8334
- package/schemas/audit.schema.json +0 -226
- package/schemas/capability.schema.json +0 -113
- package/schemas/envelope.schema.json +0 -162
- package/schemas/memory.schema.json +0 -51
- package/schemas/observation.schema.json +0 -113
- package/schemas/plan.schema.json +0 -208
- package/schemas/progress.schema.json +0 -126
- package/schemas/proposal.schema.json +0 -68
- package/schemas/session.schema.json +0 -82
- package/schemas/tool.schema.json +0 -94
- package/server.d.ts.map +0 -1
- package/server.js +0 -28877
- package/umd-example.html +0 -18
- package/umd.d.ts.map +0 -1
- /package/{bun.d.ts → dist/bun.d.ts} +0 -0
- /package/{cli.d.ts → dist/cli.d.ts} +0 -0
- /package/{companion.d.ts → dist/companion.d.ts} +0 -0
- /package/{crossbrowser.d.ts → dist/crossbrowser.d.ts} +0 -0
- /package/{dashdone.d.ts → dist/dashdone.d.ts} +0 -0
- /package/{deno.d.ts → dist/deno.d.ts} +0 -0
- /package/{hardening.d.ts → dist/hardening.d.ts} +0 -0
- /package/{headless.d.ts → dist/headless.d.ts} +0 -0
- /package/{http.d.ts → dist/http.d.ts} +0 -0
- /package/{index.d.ts → dist/index.d.ts} +0 -0
- /package/{mcp.d.ts → dist/mcp.d.ts} +0 -0
- /package/{memory.d.ts → dist/memory.d.ts} +0 -0
- /package/{node.d.ts → dist/node.d.ts} +0 -0
- /package/{pack.d.ts → dist/pack.d.ts} +0 -0
- /package/{policy.d.ts → dist/policy.d.ts} +0 -0
- /package/{progress.d.ts → dist/progress.d.ts} +0 -0
- /package/{protocol.d.ts → dist/protocol.d.ts} +0 -0
- /package/{server.d.ts → dist/server.d.ts} +0 -0
- /package/{umd.d.ts → dist/umd.d.ts} +0 -0
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
cropshot,
|
|
4
|
+
frameocr,
|
|
5
|
+
groundshot,
|
|
6
|
+
imagehashof,
|
|
7
|
+
imageocr,
|
|
8
|
+
mergelines,
|
|
9
|
+
mergeparagraphs,
|
|
10
|
+
ocrtext,
|
|
11
|
+
pairquery,
|
|
12
|
+
pairshot,
|
|
13
|
+
pdfocr,
|
|
14
|
+
proposeredactionmasks,
|
|
15
|
+
redactshot,
|
|
16
|
+
regionocr,
|
|
17
|
+
visioncacheput,
|
|
18
|
+
visioncacheserve,
|
|
19
|
+
visioncost,
|
|
20
|
+
visionshot,
|
|
21
|
+
type framegrab,
|
|
22
|
+
type maskfill,
|
|
23
|
+
type ocrread,
|
|
24
|
+
type pdfrasterize,
|
|
25
|
+
type videostateread,
|
|
26
|
+
type visionsend,
|
|
27
|
+
} from "../capture.js";
|
|
28
|
+
import {
|
|
29
|
+
framebudgetgate,
|
|
30
|
+
groundgate,
|
|
31
|
+
ocrgate,
|
|
32
|
+
pairgate,
|
|
33
|
+
redactgate,
|
|
34
|
+
regionboundsgate,
|
|
35
|
+
visioncacheexpirygate,
|
|
36
|
+
visionchoices,
|
|
37
|
+
visionconsentgate,
|
|
38
|
+
visioncostgrade,
|
|
39
|
+
visiongate,
|
|
40
|
+
visionsensitivegrade,
|
|
41
|
+
} from "../policy.js";
|
|
42
|
+
import type { ocrregion, ocrword, redactionmask, streamcursor, visiondescription } from "../types.js";
|
|
43
|
+
|
|
44
|
+
const now = 1_800_000_000_000;
|
|
45
|
+
|
|
46
|
+
/** Builds one recognized word fixture with its box and confidence. */
|
|
47
|
+
function word(text: string, box: ocrregion, confidence: number): ocrword {
|
|
48
|
+
return { text, box, confidence };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Builds one vision description fixture with labeled regions on a 400 by 800 image. */
|
|
52
|
+
function description(over: Partial<visiondescription> = {}): visiondescription {
|
|
53
|
+
return {
|
|
54
|
+
id: "d1",
|
|
55
|
+
runid: "run1",
|
|
56
|
+
stepid: "st1",
|
|
57
|
+
imageid: "cap1",
|
|
58
|
+
prompt: "Describe the checkout page.",
|
|
59
|
+
text: "A checkout page with a total and a pay button.",
|
|
60
|
+
regions: [
|
|
61
|
+
{ label: "pay button", box: { x: 160, y: 700, width: 80, height: 40 } },
|
|
62
|
+
{ label: "order total", box: { x: 40, y: 640, width: 120, height: 30 } },
|
|
63
|
+
],
|
|
64
|
+
at: now,
|
|
65
|
+
...over,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe("vision imageocr recognition", () => {
|
|
70
|
+
it("merges the recognized word boxes into lines and paragraphs by their own geometry while every word carries its confidence and the line reports the lowest one", async () => {
|
|
71
|
+
const words = [
|
|
72
|
+
word("Order", { x: 10, y: 20, width: 60, height: 20 }, 0.98),
|
|
73
|
+
word("total:", { x: 80, y: 21, width: 50, height: 19 }, 0.71),
|
|
74
|
+
word("42.50", { x: 140, y: 20, width: 60, height: 20 }, 0.95),
|
|
75
|
+
word("Pay", { x: 12, y: 90, width: 40, height: 20 }, 0.99),
|
|
76
|
+
word("now", { x: 60, y: 91, width: 50, height: 20 }, 0.88),
|
|
77
|
+
word("Thanks", { x: 14, y: 115, width: 80, height: 24 }, 1),
|
|
78
|
+
];
|
|
79
|
+
const lines = mergelines(words);
|
|
80
|
+
expect(lines.map((line) => line.text)).toEqual(["Order total: 42.50", "Pay now", "Thanks"]);
|
|
81
|
+
expect(lines[0]?.words).toBe(3);
|
|
82
|
+
expect(lines[0]?.confidence).toBe(0.71);
|
|
83
|
+
expect(lines[0]?.box).toEqual({ x: 10, y: 20, width: 190, height: 20 });
|
|
84
|
+
expect(lines[1]?.confidence).toBe(0.88);
|
|
85
|
+
expect(mergeparagraphs(lines)).toEqual(["Order total: 42.50", "Pay now Thanks"]);
|
|
86
|
+
const read: ocrread = async () => words;
|
|
87
|
+
const result = await imageocr({
|
|
88
|
+
record: { runid: "run1", stepid: "st1", imageid: "cap1" },
|
|
89
|
+
image: "data:image/png;base64,abc",
|
|
90
|
+
read,
|
|
91
|
+
id: "ocr1",
|
|
92
|
+
now,
|
|
93
|
+
});
|
|
94
|
+
expect(result.id).toBe("ocr1");
|
|
95
|
+
expect(result.imageid).toBe("cap1");
|
|
96
|
+
expect(result.words).toHaveLength(6);
|
|
97
|
+
expect(result.lines).toHaveLength(3);
|
|
98
|
+
expect(result.paragraphs).toEqual(["Order total: 42.50", "Pay now Thanks"]);
|
|
99
|
+
expect(result.text).toBe("Order total: 42.50 Pay now Thanks");
|
|
100
|
+
expect(ocrtext(" Order total:\n\n42.50 ")).toBe("Order total: 42.50");
|
|
101
|
+
await expect(
|
|
102
|
+
imageocr({ record: { runid: "run1", stepid: "st1", imageid: "cap1" }, image: " ", read, now }),
|
|
103
|
+
).rejects.toThrow(/needs its captured image/);
|
|
104
|
+
expect(ocrgate("imageocr").allowed).toBe(true);
|
|
105
|
+
expect(ocrgate("visioncall").allowed).toBe(false);
|
|
106
|
+
expect(visionchoices({ visionmodel: "see-1", visionendpoint: "https://vision.example/model" })).toEqual({
|
|
107
|
+
model: "see-1",
|
|
108
|
+
endpoint: "https://vision.example/model",
|
|
109
|
+
});
|
|
110
|
+
expect(visionchoices(undefined)).toEqual({});
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("vision regionocr bounds and pdfocr page streaming", () => {
|
|
115
|
+
it("clamps the drawn region into the viewport while the word boxes offset to absolute coordinates and pdfocr reads one page at a time through the streamcursor checkpoints", async () => {
|
|
116
|
+
const read: ocrread = async (_image, region) => [
|
|
117
|
+
{ text: "hello", box: { x: 0, y: 0, width: 50, height: 20 }, confidence: 0.9 },
|
|
118
|
+
{ text: "world", box: { x: 5, y: 2, width: 50, height: 18 }, confidence: 0.8 },
|
|
119
|
+
];
|
|
120
|
+
const clamped = await regionocr({
|
|
121
|
+
record: { runid: "run1", stepid: "st1", imageid: "cap1" },
|
|
122
|
+
image: "data:image/png;base64,abc",
|
|
123
|
+
region: { x: 380, y: 700, width: 120, height: 80 },
|
|
124
|
+
viewport: { width: 400, height: 800 },
|
|
125
|
+
read,
|
|
126
|
+
id: "ocr2",
|
|
127
|
+
now,
|
|
128
|
+
});
|
|
129
|
+
expect(clamped.words[0]?.box.x).toBe(380);
|
|
130
|
+
expect(clamped.words[0]?.box.y).toBe(700);
|
|
131
|
+
expect(clamped.words[1]?.box.x).toBe(385);
|
|
132
|
+
expect(clamped.text).toBe("hello world");
|
|
133
|
+
await expect(
|
|
134
|
+
regionocr({
|
|
135
|
+
record: { runid: "run1", stepid: "st1", imageid: "cap1" },
|
|
136
|
+
image: "data:image/png;base64,abc",
|
|
137
|
+
region: { x: 500, y: 900, width: 40, height: 30 },
|
|
138
|
+
viewport: { width: 400, height: 800 },
|
|
139
|
+
read,
|
|
140
|
+
now,
|
|
141
|
+
}),
|
|
142
|
+
).rejects.toThrow(/falls entirely outside/);
|
|
143
|
+
const inside = regionboundsgate({
|
|
144
|
+
region: { x: 10, y: 10, width: 100, height: 50 },
|
|
145
|
+
viewport: { width: 400, height: 800 },
|
|
146
|
+
});
|
|
147
|
+
expect(inside.allowed).toBe(true);
|
|
148
|
+
const crossing = regionboundsgate({
|
|
149
|
+
region: { x: 350, y: 10, width: 100, height: 50 },
|
|
150
|
+
viewport: { width: 400, height: 800 },
|
|
151
|
+
});
|
|
152
|
+
expect(crossing.allowed).toBe(true);
|
|
153
|
+
expect(crossing.reason).toContain("clamps");
|
|
154
|
+
expect(
|
|
155
|
+
regionboundsgate({ region: { x: 450, y: 10, width: 100, height: 50 }, viewport: { width: 400, height: 800 } })
|
|
156
|
+
.allowed,
|
|
157
|
+
).toBe(false);
|
|
158
|
+
expect(
|
|
159
|
+
regionboundsgate({ region: { x: 10, y: 10, width: 0, height: 50 }, viewport: { width: 400, height: 800 } })
|
|
160
|
+
.allowed,
|
|
161
|
+
).toBe(false);
|
|
162
|
+
const rasterized: string[] = [];
|
|
163
|
+
const rasterize: pdfrasterize = async (documentid, page) => {
|
|
164
|
+
rasterized.push(`${documentid}:p${page}`);
|
|
165
|
+
return `data:image/png;base64,page${page}`;
|
|
166
|
+
};
|
|
167
|
+
const checkpoints: streamcursor[] = [];
|
|
168
|
+
const pass = await pdfocr({
|
|
169
|
+
record: { runid: "run1", stepid: "st1", documentid: "scan-1" },
|
|
170
|
+
pages: 3,
|
|
171
|
+
rasterize,
|
|
172
|
+
read,
|
|
173
|
+
checkpoint: (cursor) => {
|
|
174
|
+
checkpoints.push({ ...cursor });
|
|
175
|
+
},
|
|
176
|
+
now,
|
|
177
|
+
});
|
|
178
|
+
expect(rasterized).toEqual(["scan-1:p1", "scan-1:p2", "scan-1:p3"]);
|
|
179
|
+
expect(pass.pagesread).toBe(3);
|
|
180
|
+
expect(pass.results.map((result) => result.imageid)).toEqual(["scan-1:p1", "scan-1:p2", "scan-1:p3"]);
|
|
181
|
+
expect(checkpoints[checkpoints.length - 1]).toEqual({ pipelineid: "scan-1", offset: 3, chunk: 3, updatedat: now });
|
|
182
|
+
const resumed = await pdfocr({
|
|
183
|
+
record: { runid: "run1", stepid: "st1", documentid: "scan-1" },
|
|
184
|
+
pages: 3,
|
|
185
|
+
rasterize,
|
|
186
|
+
read,
|
|
187
|
+
...(checkpoints[0] !== undefined ? { cursor: checkpoints[0] } : {}),
|
|
188
|
+
now,
|
|
189
|
+
});
|
|
190
|
+
expect(resumed.skipped).toBe(1);
|
|
191
|
+
expect(resumed.pagesread).toBe(2);
|
|
192
|
+
expect(rasterized).toHaveLength(5);
|
|
193
|
+
await expect(
|
|
194
|
+
pdfocr({
|
|
195
|
+
record: { runid: "run1", stepid: "st1", documentid: "scan-1" },
|
|
196
|
+
pages: 3,
|
|
197
|
+
rasterize,
|
|
198
|
+
read,
|
|
199
|
+
cursor: { pipelineid: "scan-other", offset: 1, chunk: 1, updatedat: now },
|
|
200
|
+
now,
|
|
201
|
+
}),
|
|
202
|
+
).rejects.toThrow(/another document/);
|
|
203
|
+
expect(ocrgate("regionocr").allowed).toBe(true);
|
|
204
|
+
expect(ocrgate("pdfocr").allowed).toBe(true);
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
describe("vision frameocr pause enforcement", () => {
|
|
209
|
+
it("refuses the frame of a playing video before any pixel moves while a paused video seeks to its reviewed position and reads the frame", async () => {
|
|
210
|
+
const words = [word("chapter", { x: 10, y: 10, width: 90, height: 20 }, 0.93)];
|
|
211
|
+
const read: ocrread = async () => words;
|
|
212
|
+
const states: Array<{ paused: boolean; positionms: number; durationms: number }> = [
|
|
213
|
+
{ paused: false, positionms: 4_000, durationms: 60_000 },
|
|
214
|
+
{ paused: true, positionms: 12_000, durationms: 60_000 },
|
|
215
|
+
];
|
|
216
|
+
let stateindex = 0;
|
|
217
|
+
const stateread: videostateread = async () => states[Math.min(stateindex, states.length - 1)]!;
|
|
218
|
+
const grabs: number[] = [];
|
|
219
|
+
const grab: framegrab = async (_selector, positionms) => {
|
|
220
|
+
grabs.push(positionms);
|
|
221
|
+
return { image: "data:image/png;base64,frame", width: 640, height: 360 };
|
|
222
|
+
};
|
|
223
|
+
await expect(
|
|
224
|
+
frameocr({
|
|
225
|
+
record: { runid: "run1", stepid: "st1" },
|
|
226
|
+
frame: { runid: "run1", stepid: "st1", selector: "video.tutorial", positionms: 12_000, at: now },
|
|
227
|
+
state: stateread,
|
|
228
|
+
grab,
|
|
229
|
+
read,
|
|
230
|
+
now,
|
|
231
|
+
}),
|
|
232
|
+
).rejects.toThrow(/refuses a moving picture/);
|
|
233
|
+
expect(grabs).toHaveLength(0);
|
|
234
|
+
stateindex = 1;
|
|
235
|
+
const played = await frameocr({
|
|
236
|
+
record: { runid: "run1", stepid: "st1" },
|
|
237
|
+
frame: { runid: "run1", stepid: "st1", selector: "video.tutorial", positionms: 12_000, at: now },
|
|
238
|
+
state: stateread,
|
|
239
|
+
grab,
|
|
240
|
+
read,
|
|
241
|
+
now,
|
|
242
|
+
});
|
|
243
|
+
expect(grabs).toEqual([12_000]);
|
|
244
|
+
expect(played.result.words[0]?.text).toBe("chapter");
|
|
245
|
+
expect(played.frame.positionms).toBe(12_000);
|
|
246
|
+
await expect(
|
|
247
|
+
frameocr({
|
|
248
|
+
record: { runid: "run1", stepid: "st1" },
|
|
249
|
+
frame: { runid: "run1", stepid: "st1", selector: "video.tutorial", positionms: 90_000, at: now },
|
|
250
|
+
state: stateread,
|
|
251
|
+
grab,
|
|
252
|
+
read,
|
|
253
|
+
now,
|
|
254
|
+
}),
|
|
255
|
+
).rejects.toThrow(/outside the video/);
|
|
256
|
+
expect(ocrgate("frameocr").allowed).toBe(true);
|
|
257
|
+
expect(framebudgetgate({ waitms: 800, budget: 1500 }).allowed).toBe(true);
|
|
258
|
+
expect(framebudgetgate({ waitms: 1800, budget: 1500 }).allowed).toBe(false);
|
|
259
|
+
expect(framebudgetgate({ waitms: 800 }).allowed).toBe(true);
|
|
260
|
+
expect(visionconsentgate({ frames: true, consented: false }).allowed).toBe(false);
|
|
261
|
+
expect(visionconsentgate({ frames: true, consented: true }).allowed).toBe(true);
|
|
262
|
+
expect(visionconsentgate({ frames: false, consented: false }).allowed).toBe(true);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
describe("vision cropshot scaling and pairshot alignment", () => {
|
|
267
|
+
it("scales the element crop into image pixels with the device pixel ratio awareness while the pair aligns with the nearest same viewport dom snapshot", () => {
|
|
268
|
+
const crop = cropshot({
|
|
269
|
+
image: { width: 1280, height: 800 },
|
|
270
|
+
bounds: { x: 100, y: 300, width: 200, height: 80 },
|
|
271
|
+
pixelratio: 2,
|
|
272
|
+
});
|
|
273
|
+
expect(crop.cssbounds).toEqual({ x: 100, y: 300, width: 200, height: 80 });
|
|
274
|
+
expect(crop.crop).toEqual({ x: 200, y: 600, width: 400, height: 160 });
|
|
275
|
+
expect(crop.width).toBe(400);
|
|
276
|
+
expect(crop.height).toBe(160);
|
|
277
|
+
expect(crop.ratio).toBe(2);
|
|
278
|
+
const clamped = cropshot({
|
|
279
|
+
image: { width: 1280, height: 800 },
|
|
280
|
+
bounds: { x: 600, y: 380, width: 120, height: 60 },
|
|
281
|
+
pixelratio: 2,
|
|
282
|
+
});
|
|
283
|
+
expect(clamped.crop).toEqual({ x: 1200, y: 760, width: 80, height: 40 });
|
|
284
|
+
expect(() =>
|
|
285
|
+
cropshot({
|
|
286
|
+
image: { width: 1280, height: 800 },
|
|
287
|
+
bounds: { x: 1400, y: 900, width: 60, height: 30 },
|
|
288
|
+
pixelratio: 2,
|
|
289
|
+
}),
|
|
290
|
+
).toThrow(/fall entirely outside/);
|
|
291
|
+
const paired = pairshot({
|
|
292
|
+
runid: "run1",
|
|
293
|
+
stepid: "st1",
|
|
294
|
+
image: { id: "cap1", capturetime: now, viewport: { width: 640, height: 400 } },
|
|
295
|
+
snapshots: [
|
|
296
|
+
{ id: "dom-older", capturetime: now - 900, viewport: { width: 640, height: 400 } },
|
|
297
|
+
{ id: "dom-nearest", capturetime: now - 80, viewport: { width: 640, height: 400 } },
|
|
298
|
+
{ id: "dom-other-viewport", capturetime: now - 10, viewport: { width: 320, height: 200 } },
|
|
299
|
+
],
|
|
300
|
+
id: "pair1",
|
|
301
|
+
now,
|
|
302
|
+
});
|
|
303
|
+
expect(paired.pair?.domsnapshotid).toBe("dom-nearest");
|
|
304
|
+
expect(paired.pair?.imageid).toBe("cap1");
|
|
305
|
+
expect(paired.pair?.viewport).toEqual({ width: 640, height: 400 });
|
|
306
|
+
expect(paired.reason).toContain("80 milliseconds");
|
|
307
|
+
const unmatched = pairshot({
|
|
308
|
+
runid: "run1",
|
|
309
|
+
stepid: "st1",
|
|
310
|
+
image: { id: "cap2", capturetime: now, viewport: { width: 640, height: 400 } },
|
|
311
|
+
snapshots: [{ id: "dom-other-viewport", capturetime: now - 10, viewport: { width: 320, height: 200 } }],
|
|
312
|
+
now,
|
|
313
|
+
});
|
|
314
|
+
expect(unmatched.pair).toBeUndefined();
|
|
315
|
+
expect(unmatched.skipped).toBe("snapshot");
|
|
316
|
+
expect(pairgate({ snapshot: { runid: "run1" }, runid: "run1" }).allowed).toBe(true);
|
|
317
|
+
expect(pairgate({ snapshot: { runid: "run2" }, runid: "run1" }).allowed).toBe(false);
|
|
318
|
+
const query = pairquery({
|
|
319
|
+
query: "Order total",
|
|
320
|
+
ocrtextvalue: "The Order total: 42.50 appears once.",
|
|
321
|
+
domtext: "Checkout summary shows the order total row.",
|
|
322
|
+
});
|
|
323
|
+
expect(query.matches.map((match) => match.source).sort()).toEqual(["dom", "image"]);
|
|
324
|
+
expect(query.matches[0]?.excerpt).toContain("order total");
|
|
325
|
+
expect(
|
|
326
|
+
pairquery({ query: "missing needle", ocrtextvalue: "nothing here", domtext: "nothing there" }).matches,
|
|
327
|
+
).toHaveLength(0);
|
|
328
|
+
expect(() => pairquery({ query: " ", ocrtextvalue: "", domtext: "" })).toThrow(/non-empty string/);
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
describe("vision redactshot masking and audit records", () => {
|
|
333
|
+
it("applies the redactionmask through the fill seam before any sharing while the proposals derive from the sensitive field shapes and the audit summary records the covered regions", async () => {
|
|
334
|
+
const filled: ocrregion[] = [];
|
|
335
|
+
const fill: maskfill = async (image, regions) => {
|
|
336
|
+
filled.push(...regions.map((region) => ({ ...region })));
|
|
337
|
+
return `${image}:masked`;
|
|
338
|
+
};
|
|
339
|
+
const mask: redactionmask = {
|
|
340
|
+
id: "mask1",
|
|
341
|
+
runid: "run1",
|
|
342
|
+
captureid: "cap1",
|
|
343
|
+
regions: [
|
|
344
|
+
{ x: 10, y: 20, width: 100, height: 30 },
|
|
345
|
+
{ x: 380, y: 700, width: 120, height: 40 },
|
|
346
|
+
],
|
|
347
|
+
reason: "the payment card digits stay on the device",
|
|
348
|
+
source: "userdrawn",
|
|
349
|
+
at: now,
|
|
350
|
+
};
|
|
351
|
+
const masked = await redactshot({ image: "data:image/png;base64,abc", width: 400, height: 800, mask, fill });
|
|
352
|
+
expect(masked.image).toBe("data:image/png;base64,abc:masked");
|
|
353
|
+
expect(masked.regions).toBe(2);
|
|
354
|
+
expect(masked.reason).toBe("the payment card digits stay on the device");
|
|
355
|
+
expect(filled[1]).toEqual({ x: 380, y: 700, width: 20, height: 40 });
|
|
356
|
+
expect(masked.summary).toContain("covered 2 regions");
|
|
357
|
+
expect(masked.summary).toContain(mask.reason);
|
|
358
|
+
await expect(
|
|
359
|
+
redactshot({ image: "data:image/png;base64,abc", width: 400, height: 800, mask: { ...mask, regions: [] }, fill }),
|
|
360
|
+
).rejects.toThrow(/masks nothing/);
|
|
361
|
+
await expect(
|
|
362
|
+
redactshot({
|
|
363
|
+
image: "data:image/png;base64,abc",
|
|
364
|
+
width: 400,
|
|
365
|
+
height: 800,
|
|
366
|
+
mask: { ...mask, regions: [{ x: 900, y: 900, width: 10, height: 10 }] },
|
|
367
|
+
fill,
|
|
368
|
+
}),
|
|
369
|
+
).rejects.toThrow(/falls outside the capture/);
|
|
370
|
+
const proposals = proposeredactionmasks({
|
|
371
|
+
runid: "run1",
|
|
372
|
+
captureid: "cap1",
|
|
373
|
+
fields: [
|
|
374
|
+
{ name: "cardnumber", rect: { x: 10, y: 20, width: 120, height: 24 } },
|
|
375
|
+
{ name: "quantity", rect: { x: 10, y: 60, width: 60, height: 24 } },
|
|
376
|
+
{ name: "password", rect: { x: 10, y: 100, width: 90, height: 24 } },
|
|
377
|
+
],
|
|
378
|
+
now,
|
|
379
|
+
});
|
|
380
|
+
expect(proposals.map((proposal) => proposal.regions[0]?.x)).toEqual([10, 10]);
|
|
381
|
+
expect(proposals.every((proposal) => proposal.source === "fieldshape" && proposal.id.trim() !== "")).toBe(true);
|
|
382
|
+
expect(proposals[0]?.reason).toContain("cardnumber");
|
|
383
|
+
expect(redactgate({ redacted: true, destination: "clipboard" }).allowed).toBe(true);
|
|
384
|
+
expect(redactgate({ redacted: false, destination: "download" }).allowed).toBe(false);
|
|
385
|
+
expect(redactgate({ redacted: false, destination: "session memory" }).allowed).toBe(true);
|
|
386
|
+
expect(
|
|
387
|
+
visiongate({ endpoint: "https://vision.example/model", model: "see-1", granted: ["https://vision.example"] })
|
|
388
|
+
.allowed,
|
|
389
|
+
).toBe(true);
|
|
390
|
+
expect(
|
|
391
|
+
visiongate({ endpoint: "https://outside.example/model", model: "see-1", granted: ["https://vision.example"] })
|
|
392
|
+
.allowed,
|
|
393
|
+
).toBe(false);
|
|
394
|
+
expect(visiongate({ endpoint: "", model: "", granted: ["https://vision.example"] }).allowed).toBe(false);
|
|
395
|
+
expect(visionsensitivegrade("visionshot").allowed).toBe(true);
|
|
396
|
+
expect(visionsensitivegrade("imageocr").allowed).toBe(false);
|
|
397
|
+
});
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
describe("vision groundshot ranking and visioncache hits", () => {
|
|
401
|
+
it("ranks the grounding candidates by the averaged text and geometry scores while the visioncache serves the repeated image hash without a model call under the user retention", async () => {
|
|
402
|
+
const elements = [
|
|
403
|
+
{ selector: "button.pay", text: "Pay now with card", rect: { x: 80, y: 350, width: 40, height: 20 } },
|
|
404
|
+
{ selector: "button.cancel", text: "Cancel the order", rect: { x: 20, y: 350, width: 40, height: 20 } },
|
|
405
|
+
{ selector: "span.total", text: "Order total 42.50", rect: { x: 20, y: 320, width: 60, height: 15 } },
|
|
406
|
+
{ selector: "footer", text: "Shop footer links", rect: { x: 0, y: 780, width: 400, height: 20 } },
|
|
407
|
+
];
|
|
408
|
+
const grounded = groundshot({
|
|
409
|
+
description: description(),
|
|
410
|
+
elements,
|
|
411
|
+
pixelratio: 2,
|
|
412
|
+
record: { runid: "run1", stepid: "st1" },
|
|
413
|
+
id: "g1",
|
|
414
|
+
now,
|
|
415
|
+
});
|
|
416
|
+
expect(grounded.descriptionid).toBe("d1");
|
|
417
|
+
expect(grounded.matches[0]?.selector).toBe("span.total");
|
|
418
|
+
expect(grounded.matches[0]?.label).toBe("order total");
|
|
419
|
+
expect(grounded.matches[0]?.score).toBeGreaterThan(
|
|
420
|
+
grounded.matches.find((match) => match.selector === "button.pay")?.score ?? 0,
|
|
421
|
+
);
|
|
422
|
+
expect(grounded.matches.every((match) => match.score > 0)).toBe(true);
|
|
423
|
+
expect(grounded.matches.some((match) => match.selector === "footer")).toBe(false);
|
|
424
|
+
expect(groundgate("groundshot").allowed).toBe(true);
|
|
425
|
+
expect(groundgate("groundquery").allowed).toBe(false);
|
|
426
|
+
expect(() =>
|
|
427
|
+
groundshot({
|
|
428
|
+
description: { ...description(), regions: [] },
|
|
429
|
+
elements,
|
|
430
|
+
pixelratio: 1,
|
|
431
|
+
record: { runid: "run1", stepid: "st1" },
|
|
432
|
+
now,
|
|
433
|
+
}),
|
|
434
|
+
).toThrow(/needs its labeled regions/);
|
|
435
|
+
const send: visionsend = async () => ({
|
|
436
|
+
text: "A checkout page.",
|
|
437
|
+
regions: [{ label: "pay button", box: { x: 160, y: 700, width: 80, height: 40 } }],
|
|
438
|
+
});
|
|
439
|
+
const described = await visionshot({
|
|
440
|
+
record: { runid: "run1", stepid: "st1", imageid: "cap1" },
|
|
441
|
+
image: "data:image/png;base64,abc",
|
|
442
|
+
prompt: "Describe the page.",
|
|
443
|
+
send,
|
|
444
|
+
id: "d2",
|
|
445
|
+
now,
|
|
446
|
+
});
|
|
447
|
+
expect(described.prompt).toBe("Describe the page.");
|
|
448
|
+
expect(described.regions[0]?.label).toBe("pay button");
|
|
449
|
+
await expect(
|
|
450
|
+
visionshot({
|
|
451
|
+
record: { runid: "run1", stepid: "st1", imageid: "cap1" },
|
|
452
|
+
image: "data:image/png;base64,abc",
|
|
453
|
+
prompt: " ",
|
|
454
|
+
send,
|
|
455
|
+
now,
|
|
456
|
+
}),
|
|
457
|
+
).rejects.toThrow(/non-empty reviewed string/);
|
|
458
|
+
const hash = imagehashof("data:image/png;base64,abc");
|
|
459
|
+
const stored = visioncacheput({
|
|
460
|
+
entries: [],
|
|
461
|
+
entry: {
|
|
462
|
+
hash,
|
|
463
|
+
runid: "run1",
|
|
464
|
+
imageid: "cap1",
|
|
465
|
+
kind: "ocr",
|
|
466
|
+
ocr: {
|
|
467
|
+
id: "ocr1",
|
|
468
|
+
runid: "run1",
|
|
469
|
+
stepid: "st1",
|
|
470
|
+
imageid: "cap1",
|
|
471
|
+
text: "Order total 42.50",
|
|
472
|
+
words: [],
|
|
473
|
+
lines: [],
|
|
474
|
+
paragraphs: [],
|
|
475
|
+
at: now,
|
|
476
|
+
},
|
|
477
|
+
hits: 0,
|
|
478
|
+
at: now,
|
|
479
|
+
},
|
|
480
|
+
});
|
|
481
|
+
expect(stored).toHaveLength(1);
|
|
482
|
+
const replaced = visioncacheput({
|
|
483
|
+
entries: stored,
|
|
484
|
+
entry: {
|
|
485
|
+
hash,
|
|
486
|
+
runid: "run1",
|
|
487
|
+
imageid: "cap1",
|
|
488
|
+
kind: "ocr",
|
|
489
|
+
ocr: {
|
|
490
|
+
id: "ocr2",
|
|
491
|
+
runid: "run1",
|
|
492
|
+
stepid: "st1",
|
|
493
|
+
imageid: "cap1",
|
|
494
|
+
text: "Order total 42.50",
|
|
495
|
+
words: [],
|
|
496
|
+
lines: [],
|
|
497
|
+
paragraphs: [],
|
|
498
|
+
at: now,
|
|
499
|
+
},
|
|
500
|
+
hits: 3,
|
|
501
|
+
at: now,
|
|
502
|
+
},
|
|
503
|
+
});
|
|
504
|
+
expect(replaced).toHaveLength(1);
|
|
505
|
+
expect(replaced[0]?.hits).toBe(3);
|
|
506
|
+
const served = visioncacheserve({ entries: replaced, hash, kind: "ocr", now: now + 1000 });
|
|
507
|
+
expect(served.entry?.hits).toBe(4);
|
|
508
|
+
expect(served.entry?.ocr?.id).toBe("ocr2");
|
|
509
|
+
expect(served.expired).toBe(0);
|
|
510
|
+
const missed = visioncacheserve({
|
|
511
|
+
entries: replaced,
|
|
512
|
+
hash: imagehashof("data:image/png;base64,other"),
|
|
513
|
+
kind: "ocr",
|
|
514
|
+
now: now + 1000,
|
|
515
|
+
});
|
|
516
|
+
expect(missed.entry).toBeUndefined();
|
|
517
|
+
const expired = visioncacheserve({ entries: replaced, hash, kind: "ocr", now: now + 5000, retention: 4000 });
|
|
518
|
+
expect(expired.entry).toBeUndefined();
|
|
519
|
+
expect(expired.expired).toBe(1);
|
|
520
|
+
expect(expired.entries).toHaveLength(0);
|
|
521
|
+
expect(visioncacheserve({ entries: replaced, hash, kind: "ocr", now: now + 5000 }).entry?.hits).toBe(4);
|
|
522
|
+
expect(visioncacheexpirygate({ entryat: now, now: now + 5000, retention: 4000 }).allowed).toBe(false);
|
|
523
|
+
expect(visioncacheexpirygate({ entryat: now, now: now + 1000, retention: 4000 }).allowed).toBe(true);
|
|
524
|
+
expect(visioncacheexpirygate({ entryat: now, now: now + 5000 }).allowed).toBe(true);
|
|
525
|
+
const cost = visioncost({
|
|
526
|
+
runid: "run1",
|
|
527
|
+
calls: [
|
|
528
|
+
{ kind: "imageocr", model: true, at: now },
|
|
529
|
+
{ kind: "pairshot", model: false, at: now },
|
|
530
|
+
{ kind: "visionshot", model: true, at: now },
|
|
531
|
+
],
|
|
532
|
+
});
|
|
533
|
+
expect(cost.calls).toBe(3);
|
|
534
|
+
expect(cost.modelcalls).toBe(2);
|
|
535
|
+
expect(cost.units).toBe(2);
|
|
536
|
+
expect(cost.description).toContain("2 riding the configured model endpoint");
|
|
537
|
+
expect(visioncostgrade("visioncost").allowed).toBe(true);
|
|
538
|
+
expect(visioncostgrade("shipmetrics").allowed).toBe(false);
|
|
539
|
+
});
|
|
540
|
+
});
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
vsixcontenttypes,
|
|
4
|
+
vsixextensionhost,
|
|
5
|
+
vsixnameof,
|
|
6
|
+
vsixpackassemble,
|
|
7
|
+
vsixpackentriesof,
|
|
8
|
+
vsixpackmanifest,
|
|
9
|
+
vsixpackmanifestfields,
|
|
10
|
+
vsixmarketplacemetadatacheck,
|
|
11
|
+
vsixvsixmanifest,
|
|
12
|
+
vsixwebviewpage,
|
|
13
|
+
vsixzipof,
|
|
14
|
+
} from "../pack.js";
|
|
15
|
+
|
|
16
|
+
const esmbytes = new Uint8Array([101, 115, 109, 32, 98, 111, 100, 121]);
|
|
17
|
+
|
|
18
|
+
describe("vsixpack", () => {
|
|
19
|
+
it("names the artifact with the release version", () => {
|
|
20
|
+
expect(vsixnameof("1.1.87")).toBe("devthink-vscode-1.1.87.vsix");
|
|
21
|
+
expect(() => vsixnameof("not-a-version")).toThrow();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("declares the extension capabilities and commands with no telemetry and no network default", () => {
|
|
25
|
+
const fields = vsixpackmanifestfields("1.1.87");
|
|
26
|
+
expect(fields.name).toBe("devthink");
|
|
27
|
+
expect(fields.publisher).toBe("wenathlan");
|
|
28
|
+
expect(fields.version).toBe("1.1.87");
|
|
29
|
+
expect(fields.engines.vscode ?? "").not.toBe("");
|
|
30
|
+
expect(fields.commands.map((command) => command.command)).toEqual(["devthink.openbridge", "devthink.settings"]);
|
|
31
|
+
expect(fields.telemetry).toBe("off");
|
|
32
|
+
expect(fields.relayurldefault).toBe("");
|
|
33
|
+
expect(fields.configuration[0]?.default).toBe("");
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("assembles the vsix package with the manifest at the root and the webview inside the extension folder", () => {
|
|
37
|
+
const output = vsixpackassemble({ version: "1.1.87", esmbundlebytes: esmbytes });
|
|
38
|
+
expect(output.archive.name).toBe("devthink-vscode-1.1.87.vsix");
|
|
39
|
+
expect(output.entries).toEqual([
|
|
40
|
+
"extension.vsixmanifest",
|
|
41
|
+
"[Content_Types].xml",
|
|
42
|
+
"extension/package.json",
|
|
43
|
+
"extension/extensionhost.js",
|
|
44
|
+
"extension/webview/bridge.html",
|
|
45
|
+
"extension/index.js",
|
|
46
|
+
]);
|
|
47
|
+
expect(vsixpackentriesof(output)).toEqual(output.entries);
|
|
48
|
+
expect(output.archive.bytes.length).toBeGreaterThan(500);
|
|
49
|
+
expect(() => vsixpackassemble({ version: "1.1.87", esmbundlebytes: new Uint8Array([]) })).toThrow();
|
|
50
|
+
expect(() => vsixpackassemble({ version: "not-a-version", esmbundlebytes: esmbytes })).toThrow();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("reuses the library esm build inside the webview unchanged", () => {
|
|
54
|
+
const output = vsixpackassemble({ version: "1.1.87", esmbundlebytes: esmbytes });
|
|
55
|
+
const files = vsixzipof([{ path: "extension/index.js", bytes: Buffer.from(esmbytes) }]);
|
|
56
|
+
expect(files.length).toBeGreaterThanOrEqual(esmbytes.length);
|
|
57
|
+
/* the esm bytes ride the archive through the zip builder the package carries */
|
|
58
|
+
const body = Buffer.from(output.archive.bytes);
|
|
59
|
+
expect(body.subarray(0, 4).readUInt32LE(0)).toBe(0x04034b50);
|
|
60
|
+
expect(body.includes(Buffer.from(esmbytes))).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("keeps the consent gates inside the webview ui and connects only through the user configured url", () => {
|
|
64
|
+
const page = vsixwebviewpage("/relay");
|
|
65
|
+
expect(page).toContain("Consent gate");
|
|
66
|
+
expect(page).toContain("offline: no relay url configured");
|
|
67
|
+
expect(page).toContain("Pair after review");
|
|
68
|
+
expect(page).not.toMatch(/wss:\/\/[^"<\s]+/);
|
|
69
|
+
expect(page).not.toContain("https://");
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("runs the chatbridge surface in a webview panel through the extension host", () => {
|
|
73
|
+
const host = vsixextensionhost();
|
|
74
|
+
expect(host).toContain("createWebviewPanel");
|
|
75
|
+
expect(host).toContain("devthink.openbridge");
|
|
76
|
+
expect(host).toContain("devthink.settings");
|
|
77
|
+
expect(host).toContain("enableScripts: true");
|
|
78
|
+
/* the host opens no telemetry logger and names no endpoint of its own — the header comment states the no telemetry posture */
|
|
79
|
+
expect(host).not.toContain("createTelemetryLogger");
|
|
80
|
+
expect(host).toContain("no telemetry");
|
|
81
|
+
expect(host).not.toMatch(/https?:\/\/(?!github\.com)[^"'\s]+/);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("carries the content types and the vsix manifest of a plain zip based vsix", () => {
|
|
85
|
+
const fields = vsixpackmanifestfields("1.1.87");
|
|
86
|
+
const manifest = vsixvsixmanifest(fields, ["extension/package.json"]);
|
|
87
|
+
expect(manifest).toContain('<PackageManifest Version="2.0.0"');
|
|
88
|
+
expect(manifest).toContain(`Id="devthink" Version="1.1.87" Publisher="wenathlan"`);
|
|
89
|
+
expect(manifest).toContain("Microsoft.VisualStudio.Code.Engine");
|
|
90
|
+
expect(manifest).not.toContain("marketplace");
|
|
91
|
+
const contenttypes = vsixcontenttypes();
|
|
92
|
+
expect(contenttypes).toContain('<Default Extension="vsixmanifest"');
|
|
93
|
+
expect(contenttypes).toContain('<Default Extension="json"');
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("builds a stored zip archive every zip reader resolves without a scan", () => {
|
|
97
|
+
const archive = vsixzipof([
|
|
98
|
+
{ path: "a.txt", bytes: Buffer.from("devthink", "utf8") },
|
|
99
|
+
{ path: "b/c.txt", bytes: Buffer.from("bridge", "utf8") },
|
|
100
|
+
]);
|
|
101
|
+
expect(archive.readUInt32LE(0)).toBe(0x04034b50);
|
|
102
|
+
/* the end of central directory record counts both entries and points at the central directory */
|
|
103
|
+
const end = archive.subarray(archive.length - 22);
|
|
104
|
+
expect(end.readUInt32LE(0)).toBe(0x06054b50);
|
|
105
|
+
expect(end.readUInt16LE(8)).toBe(2);
|
|
106
|
+
const centralstart = end.readUInt32LE(16);
|
|
107
|
+
expect(archive.subarray(centralstart, centralstart + 4).readUInt32LE(0)).toBe(0x02014b50);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("validates the marketplace metadata of the package fields", () => {
|
|
111
|
+
const fields = vsixpackmanifestfields("1.1.87");
|
|
112
|
+
const manifesttext = vsixpackmanifest(fields);
|
|
113
|
+
expect(vsixmarketplacemetadatacheck({ fields, manifesttext }).ok).toBe(true);
|
|
114
|
+
/* a missing identity, a telemetry statement, a network default, a marketplace endpoint or a download url each fail the check */
|
|
115
|
+
expect(vsixmarketplacemetadatacheck({ fields: { ...fields, name: "" }, manifesttext }).ok).toBe(false);
|
|
116
|
+
expect(vsixmarketplacemetadatacheck({ fields: { ...fields, publisher: "" }, manifesttext }).ok).toBe(false);
|
|
117
|
+
expect(vsixmarketplacemetadatacheck({ fields: { ...fields, version: "latest" }, manifesttext }).ok).toBe(false);
|
|
118
|
+
expect(
|
|
119
|
+
vsixmarketplacemetadatacheck({
|
|
120
|
+
fields: { ...fields, telemetry: "off" as const, relayurldefault: "wss://example.invalid/relay" as "" },
|
|
121
|
+
manifesttext,
|
|
122
|
+
}).ok,
|
|
123
|
+
).toBe(false);
|
|
124
|
+
expect(
|
|
125
|
+
vsixmarketplacemetadatacheck({
|
|
126
|
+
fields,
|
|
127
|
+
manifesttext: `${manifesttext} https://marketplace.visualstudio.com/items`,
|
|
128
|
+
}).ok,
|
|
129
|
+
).toBe(false);
|
|
130
|
+
expect(
|
|
131
|
+
vsixmarketplacemetadatacheck({
|
|
132
|
+
fields,
|
|
133
|
+
manifesttext: `${manifesttext} https://example.invalid/download/devthink.vsix`,
|
|
134
|
+
}).ok,
|
|
135
|
+
).toBe(false);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("carries no vendor marketplace url and no download url anywhere in the assembled package", () => {
|
|
139
|
+
const output = vsixpackassemble({ version: "1.1.87", esmbundlebytes: esmbytes });
|
|
140
|
+
const text = Buffer.from(output.archive.bytes).toString("utf8");
|
|
141
|
+
expect(text).not.toMatch(/marketplace\.visualstudio\.com|clients2\.google\.com|visualstudio\.gallery/i);
|
|
142
|
+
expect(text).not.toMatch(/https?:\/\/[^\s"<>]*(download|installer|update)[^\s"<>]*/i);
|
|
143
|
+
expect(text).not.toContain("clients2.google.com");
|
|
144
|
+
});
|
|
145
|
+
});
|