@lobu/runtime-server 22.0.0 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalogs/connectors.json +1353 -181
- package/dist/catalogs/skills.json +1 -1
- package/dist/component.json +1 -1
- package/dist/connectors/README.md +14 -1
- package/dist/connectors/google_gmail.ts +1 -0
- package/dist/connectors/jira-helpers.ts +313 -0
- package/dist/connectors/jira.ts +29 -221
- package/dist/connectors/linkedin-browser-script.ts +13 -0
- package/dist/connectors/linkedin-draft-page.js +104 -0
- package/dist/connectors/linkedin-drafts.ts +73 -0
- package/dist/connectors/linkedin-identity.ts +32 -0
- package/dist/connectors/linkedin-inbox-adapter.js +1185 -0
- package/dist/connectors/linkedin-inbox.ts +156 -0
- package/dist/connectors/linkedin-like.ts +114 -0
- package/dist/connectors/linkedin-takeout.ts +577 -0
- package/dist/connectors/linkedin.ts +3998 -0
- package/dist/connectors/takeout-utils.ts +240 -0
- package/dist/connectors/tiktok-inbox-page.js +9 -9
- package/dist/connectors/tiktok-web-page.js +21 -15
- package/dist/connectors/tiktok_web.ts +63 -30
- package/dist/connectors/whatsapp-web-adapter.js +64 -564
- package/dist/connectors/whatsapp-web-helpers.ts +30 -444
- package/dist/connectors/whatsapp-web-identity.ts +20 -6
- package/dist/connectors/whatsapp_web.ts +45 -645
- package/dist/connectors/x-dm-adapter.js +556 -0
- package/dist/connectors/x-identity.ts +31 -2
- package/dist/connectors/x.ts +266 -120
- package/dist/db/migrations/20261017120000_drop_feeds_kind_virtual.sql +26 -0
- package/dist/db/migrations/20261017130000_drop_retired_dead_code_columns.sql +46 -0
- package/dist/dependencies.bun.lock +41 -41
- package/dist/guest.bundle.js +288 -148
- package/dist/owletto/dist/assets/_-3J-IrxWk.js +2 -0
- package/dist/owletto/dist/assets/{_-CEPl6gU4.js → _-DYUE3ix4.js} +3 -3
- package/dist/owletto/dist/assets/_agentId-Bn2IAjIy.js +2 -0
- package/dist/owletto/dist/assets/{_agentId-CBC_SadO.js → _agentId-Cam7Z0N7.js} +2 -2
- package/dist/owletto/dist/assets/{_automationId-YTTNksw6.js → _automationId-CXEiKVZJ.js} +2 -2
- package/dist/owletto/dist/assets/{_connectorKey._connectionId-fK0h7RAp.js → _connectorKey._connectionId-CS85l61t.js} +2 -2
- package/dist/owletto/dist/assets/{_connectorKey.index-C7TPS598.js → _connectorKey.index-clVDIyW0.js} +2 -2
- package/dist/owletto/dist/assets/{_conversationId-CEhxlTIo.js → _conversationId-Ci2k-BuN.js} +2 -2
- package/dist/owletto/dist/assets/{_deploymentId-BQfYN1U2.js → _deploymentId-BuUzGtVL.js} +2 -2
- package/dist/owletto/dist/assets/{_eventId-Cl8GSdjg.js → _eventId-B0662F8h.js} +2 -2
- package/dist/owletto/dist/assets/_name-CwWePlOl.js +2 -0
- package/dist/owletto/dist/assets/{_owner-cUtGHqSw.js → _owner-dTWkrKsK.js} +3 -3
- package/dist/owletto/dist/assets/_shape-BtS3y4M9.js +2 -0
- package/dist/owletto/dist/assets/_skillKey-Dc9m-nlR.js +2 -0
- package/dist/owletto/dist/assets/{_slug-CU1PuHB9.js → _slug-DQHutM41.js} +2 -2
- package/dist/owletto/dist/assets/_viewKey-B30ik8ma.js +2 -0
- package/dist/owletto/dist/assets/_viewKey-CUHypQnF.js +2 -0
- package/dist/owletto/dist/assets/{accept-invitation-D3OgJG4i.js → accept-invitation-w17IFTtf.js} +2 -2
- package/dist/owletto/dist/assets/{activity-route-view-DQglkhXU.js → activity-route-view-CMqYiiKM.js} +2 -2
- package/dist/owletto/dist/assets/{agent-card-controls-DzRs4Ix9.js → agent-card-controls-mnTN2AZt.js} +2 -2
- package/dist/owletto/dist/assets/{agent-client-setup-list-B2SvH7xD.js → agent-client-setup-list-Bg3hBfUV.js} +2 -2
- package/dist/owletto/dist/assets/{agent-connect-pages-B6uKvT_X.js → agent-connect-pages-B_vrqLVc.js} +2 -2
- package/dist/owletto/dist/assets/{agent-editor-form-oUnIYHd7.js → agent-editor-form-hTs7eO_U.js} +2 -2
- package/dist/owletto/dist/assets/{agent-guardrails-route-BaV1VpDh.js → agent-guardrails-route-BOrCXcAl.js} +2 -2
- package/dist/owletto/dist/assets/{agent-loading-shells-CtZJM2Gc.js → agent-loading-shells-Dqcvky-k.js} +2 -2
- package/dist/owletto/dist/assets/{agent-permissions-matrix-Ds48KFK9.js → agent-permissions-matrix-BOo_4r_4.js} +2 -2
- package/dist/owletto/dist/assets/{agent-skill-context-DPk5r08w.js → agent-skill-context-CCdvtlmH.js} +2 -2
- package/dist/owletto/dist/assets/{agent-skills-route-DYpB5H1J.js → agent-skills-route-BUxIswuq.js} +2 -2
- package/dist/owletto/dist/assets/{agent-subpage-shell-B3K_jPYR.js → agent-subpage-shell-D92imvSt.js} +2 -2
- package/dist/owletto/dist/assets/agent-thread-BJOijbT0.js +6 -0
- package/dist/owletto/dist/assets/agents-9XjpDj_S.js +2 -0
- package/dist/owletto/dist/assets/{agents-workbench-CMyk4PI7.js → agents-workbench-Bt5QCYQ7.js} +2 -2
- package/dist/owletto/dist/assets/{app._appId-DH1Qutz6.js → app._appId-CKv7_squ.js} +2 -2
- package/dist/owletto/dist/assets/{automation-detail-content-CEDNboId.js → automation-detail-content-vUwVumXf.js} +2 -2
- package/dist/owletto/dist/assets/automations-9XjpDj_S.js +2 -0
- package/dist/owletto/dist/assets/{callback-CU-hb8JB.js → callback-BRgcL8gz.js} +2 -2
- package/dist/owletto/dist/assets/{card-DIF8SF9b.js → card-Djgp00r_.js} +2 -2
- package/dist/owletto/dist/assets/{claim-DELEa-gH.js → claim-BYUZnyMp.js} +2 -2
- package/dist/owletto/dist/assets/{clients-CpG676mg.js → clients-D3Zbjxin.js} +2 -2
- package/dist/owletto/dist/assets/{code-editor-V27Z4o3P.js → code-editor-DHeyRGZI.js} +2 -2
- package/dist/owletto/dist/assets/connect-CfsFsy9P.js +2 -0
- package/dist/owletto/dist/assets/connect._clientId-DkiC8xos.js +2 -0
- package/dist/owletto/dist/assets/connected._clientId-CIXrGOsT.js +2 -0
- package/dist/owletto/dist/assets/{connected._clientId.activity-DQWcUqKM.js → connected._clientId.activity-Ckdz6I0y.js} +2 -2
- package/dist/owletto/dist/assets/{connected._clientId.index-BC3R303A.js → connected._clientId.index-CNxafNeo.js} +2 -2
- package/dist/owletto/dist/assets/{connection-DySpRE9x.js → connection-C3XXFEqh.js} +2 -2
- package/dist/owletto/dist/assets/{connection-feed-detail-C_D68j6d.js → connection-feed-detail-CIJJLDP8.js} +2 -2
- package/dist/owletto/dist/assets/{connections-browser-Cnv2iBbZ.js → connections-browser-CKZjzRz6.js} +2 -2
- package/dist/owletto/dist/assets/{connector-provenance-CNXtzYg5.js → connector-provenance-8-d8hl10.js} +2 -2
- package/dist/owletto/dist/assets/{connector-scope-lists-BKDetNDc.js → connector-scope-lists-CadPVsI3.js} +2 -2
- package/dist/owletto/dist/assets/{connectors-BJnk1z_2.js → connectors-lvLJ30lo.js} +2 -2
- package/dist/owletto/dist/assets/{consent-BVzxrAO9.js → consent-CdSvVCgx.js} +2 -2
- package/dist/owletto/dist/assets/{conversations-table-DfZGICl7.js → conversations-table-BdXoZAfc.js} +2 -2
- package/dist/owletto/dist/assets/{core.esm-DiTpULFl.js → core.esm-CYvgWCuv.js} +2 -2
- package/dist/owletto/dist/assets/create-0xW9eXxG.js +2 -0
- package/dist/owletto/dist/assets/{create-JRFM8mMf.js → create-B0qhrrX4.js} +2 -2
- package/dist/owletto/dist/assets/create-CAfF-yI1.js +2 -0
- package/dist/owletto/dist/assets/create-automation-form-CtSSeYj3.js +3 -0
- package/dist/owletto/dist/assets/create._connectorKey-i6xxkWKV.js +2 -0
- package/dist/owletto/dist/assets/{cronstrue-DPXl0qA2.js → cronstrue-n4qGUraE.js} +2 -2
- package/dist/owletto/dist/assets/deployments-4Itm35Xc.js +2 -0
- package/dist/owletto/dist/assets/{deployments-DCQdFdwG.js → deployments-RvUdVkj3.js} +2 -2
- package/dist/owletto/dist/assets/{deployments-body-D6TAWs_6.js → deployments-body-BMp7UsMo.js} +2 -2
- package/dist/owletto/dist/assets/{device-CntqYW65.js → device-CPU6CjkJ.js} +2 -2
- package/dist/owletto/dist/assets/{device-pin-tombstone-Bz4JHHMK.js → device-pin-tombstone-CrjG7gPo.js} +2 -2
- package/dist/owletto/dist/assets/{device._deviceId-D4plO3GM.js → device._deviceId-Cs3ZmXWo.js} +2 -2
- package/dist/owletto/dist/assets/{entity-options-UJs64GdQ.js → entity-options-uXjgT7y9.js} +2 -2
- package/dist/owletto/dist/assets/{entity-page-Djz6T50J.js → entity-page-Dc0LGr5U.js} +2 -2
- package/dist/owletto/dist/assets/{entity-type-page-OznRag6A.js → entity-type-page-CwZxa0BR.js} +2 -2
- package/dist/owletto/dist/assets/{event-detail-04qOUGIP.js → event-detail-ve2d6Gbb.js} +2 -2
- package/dist/owletto/dist/assets/{events-feed-fucGZSJR.js → events-feed-BtvhZF1f.js} +2 -2
- package/dist/owletto/dist/assets/{filter-editors-DCQgGLyR.js → filter-editors-Cj-v6NtH.js} +2 -2
- package/dist/owletto/dist/assets/{full-page-spinner-KpV2FS2u.js → full-page-spinner-DqpH99np.js} +2 -2
- package/dist/owletto/dist/assets/guardrails-HFo2Ut8B.js +2 -0
- package/dist/owletto/dist/assets/identity-BigVSBCl.js +2 -0
- package/dist/owletto/dist/assets/{index-BEvyBdOU.js → index-B9uPN6FC.js} +2 -2
- package/dist/owletto/dist/assets/{index-BNh2cidi.js → index-BQVliQW1.js} +2 -2
- package/dist/owletto/dist/assets/index-BfedB4Rc.js +2 -0
- package/dist/owletto/dist/assets/index-CUw3aKBI.js +2 -0
- package/dist/owletto/dist/assets/index-CXhxUkv3.js +2 -0
- package/dist/owletto/dist/assets/{index-_oPeVpL_.js → index-Cr5hwh-C.js} +66 -66
- package/dist/owletto/dist/assets/index-CzkTV4ZQ.js +2 -0
- package/dist/owletto/dist/assets/index-DKSx8JN_.js +2 -0
- package/dist/owletto/dist/assets/{index-D3s8rhG9.js → index-DVk7Ug0d.js} +2 -2
- package/dist/owletto/dist/assets/index-DmZuKGf8.js +2 -0
- package/dist/owletto/dist/assets/{index-DCdkwg9H.js → index-Hio-HRfo.js} +2 -2
- package/dist/owletto/dist/assets/{index-BJXkc60S.js → index-ICgKp3YK.js} +2 -2
- package/dist/owletto/dist/assets/index-cPcroOH0.js +2 -0
- package/dist/owletto/dist/assets/{index-BOzuAbvi.js → index-n7sTbNv4.js} +2 -2
- package/dist/owletto/dist/assets/index-xiFpcWKv.js +2 -0
- package/dist/owletto/dist/assets/{json-diff-D_Ym3Uku.js → json-diff-DBoag_OS.js} +2 -2
- package/dist/owletto/dist/assets/{list-skeleton-Bl0OS39L.js → list-skeleton-C_GuoxAH.js} +2 -2
- package/dist/owletto/dist/assets/{login-QHdxqX-4.js → login-Ci6ixHuL.js} +2 -2
- package/dist/owletto/dist/assets/{logout-DUccApuZ.js → logout-CLGgeZB7.js} +2 -2
- package/dist/owletto/dist/assets/{member-scope-gate-CFbrq1by.js → member-scope-gate-B44INDPY.js} +2 -2
- package/dist/owletto/dist/assets/{memory-browse-page-IrIXuZTF.js → memory-browse-page-DyQiRvzl.js} +2 -2
- package/dist/owletto/dist/assets/{model-selector-CB1BDvlW.js → model-selector-dAbDgJvx.js} +2 -2
- package/dist/owletto/dist/assets/models-DpVP7ZwX.js +2 -0
- package/dist/owletto/dist/assets/{new-BWo8_Pms.js → new-BUb7AnEW.js} +2 -2
- package/dist/owletto/dist/assets/{new-B5UsLiy6.js → new-BkdULx2r.js} +2 -2
- package/dist/owletto/dist/assets/{new-DpmDXmFI.js → new-CHOr-EcI.js} +2 -2
- package/dist/owletto/dist/assets/{oauth-workspace-access-PGyTnW2K.js → oauth-workspace-access-BxsjwgdH.js} +2 -2
- package/dist/owletto/dist/assets/{organization-BH8PYlVu.js → organization-gxGeDjIV.js} +2 -2
- package/dist/owletto/dist/assets/{peek-pane-content-B2AFNTzP.js → peek-pane-content-CHoQHe2H.js} +2 -2
- package/dist/owletto/dist/assets/{permissions-CwN_YP3u.js → permissions-aPHaKLec.js} +2 -2
- package/dist/owletto/dist/assets/{proposal-review-shell-DKKGu4Ww.js → proposal-review-shell-UcZr87_Q.js} +2 -2
- package/dist/owletto/dist/assets/public-shell-ready-DcnlADmj.js +2 -0
- package/dist/owletto/dist/assets/{recent-DqdZE_TI.js → recent-D-65Pgu-.js} +2 -2
- package/dist/owletto/dist/assets/{record-activity-CvgZyCmw.js → record-activity-CBXH6v1d.js} +2 -2
- package/dist/owletto/dist/assets/{renderer-BL6VcwQH.js → renderer-mEDnVW0O.js} +2 -2
- package/dist/owletto/dist/assets/{reset-password-DAvBYkiI.js → reset-password-uX_V6bip.js} +2 -2
- package/dist/owletto/dist/assets/{runs-BWtc5rmc.js → runs-C0rCgGwX.js} +2 -2
- package/dist/owletto/dist/assets/runs-panel-Du1JMjg5.js +2 -0
- package/dist/owletto/dist/assets/{runs-table-DxIiqrSk.js → runs-table-syPJ6r6T.js} +2 -2
- package/dist/owletto/dist/assets/{settings-Crxlwps-.js → settings-9NkNA1G1.js} +2 -2
- package/dist/owletto/dist/assets/settings-E-10sLQX.js +2 -0
- package/dist/owletto/dist/assets/skills-D2Ri9b4f.js +2 -0
- package/dist/owletto/dist/assets/{source-record-panels-CRIPMWfN.js → source-record-panels-CPyzlfD1.js} +2 -2
- package/dist/owletto/dist/assets/{status-dot-C5KDvvm9.js → status-dot-B-EWMgq6.js} +2 -2
- package/dist/owletto/dist/assets/{tabs-B4Er9hJj.js → tabs-DqIIkCX-.js} +2 -2
- package/dist/owletto/dist/assets/{use-mention-search-DqEykWl_.js → use-mention-search-By8RZyKW.js} +2 -2
- package/dist/owletto/dist/assets/{view-param-filters-Cylutoy1.js → view-param-filters-CqLZQrnO.js} +2 -2
- package/dist/owletto/dist/index.html +1 -1
- package/dist/server-main.bundle.mjs +11447 -10762
- package/npm-shrinkwrap.json +160 -160
- package/package.json +1 -1
- package/vendor/connector-sdk/dist/browser-handoff-banner.d.ts +39 -0
- package/vendor/connector-sdk/dist/browser-handoff-banner.js +175 -0
- package/vendor/connector-sdk/dist/browser-source-read.d.ts +52 -0
- package/vendor/connector-sdk/dist/browser-source-read.js +107 -0
- package/vendor/connector-sdk/dist/connector-types.d.ts +10 -0
- package/vendor/connector-sdk/dist/index.d.ts +3 -0
- package/vendor/connector-sdk/dist/index.js +2 -0
- package/vendor/connector-sdk/package.json +9 -1
- package/vendor/core/dist/capabilities.d.ts +1 -1
- package/vendor/core/dist/capabilities.js +0 -1
- package/vendor/core/dist/connector-query-errors.d.ts +32 -0
- package/vendor/core/dist/connector-query-errors.js +41 -1
- package/vendor/core/dist/contracts/tools/manage-conversations.js +2 -2
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +17 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +8 -131
- package/vendor/core/dist/contracts/worker/protocol.js +9 -61
- package/vendor/core/dist/device-pin-tombstones.d.ts +44 -0
- package/vendor/core/dist/device-pin-tombstones.js +57 -0
- package/vendor/core/dist/index.d.ts +2 -1
- package/vendor/core/dist/index.js +1 -0
- package/vendor/core/dist/logger.js +10 -25
- package/vendor/core/dist/utils/diagnostic-scrubber.js +258 -21
- package/vendor/core/dist/worker/wire.d.ts +0 -10
- package/vendor/core/package.json +6 -6
- package/vendor/views/package.json +1 -1
- package/dist/owletto/dist/assets/_-C_zNZben.js +0 -2
- package/dist/owletto/dist/assets/_agentId-QQW91cHX.js +0 -2
- package/dist/owletto/dist/assets/_name-t3oBJ6Zf.js +0 -2
- package/dist/owletto/dist/assets/_shape-bo2lMVCu.js +0 -2
- package/dist/owletto/dist/assets/_skillKey-Dwm4Q4wI.js +0 -2
- package/dist/owletto/dist/assets/_viewKey-Bh6E_NxA.js +0 -2
- package/dist/owletto/dist/assets/_viewKey-D4jKl-Zf.js +0 -2
- package/dist/owletto/dist/assets/agent-thread-C8UaUTWc.js +0 -6
- package/dist/owletto/dist/assets/agents-F4mGI2oF.js +0 -2
- package/dist/owletto/dist/assets/automations-F4mGI2oF.js +0 -2
- package/dist/owletto/dist/assets/connect-B2YL6qIK.js +0 -2
- package/dist/owletto/dist/assets/connect._clientId-B6LDmDRV.js +0 -2
- package/dist/owletto/dist/assets/connected._clientId-D_HaKoI1.js +0 -2
- package/dist/owletto/dist/assets/create-CUqO6Kfk.js +0 -2
- package/dist/owletto/dist/assets/create-automation-form-_4vUkrls.js +0 -3
- package/dist/owletto/dist/assets/create-iHcxV0xd.js +0 -2
- package/dist/owletto/dist/assets/create._connectorKey-CCYLEicg.js +0 -2
- package/dist/owletto/dist/assets/deployments-ihkhK1zV.js +0 -2
- package/dist/owletto/dist/assets/guardrails-molekzAF.js +0 -2
- package/dist/owletto/dist/assets/identity-BJcaG66g.js +0 -2
- package/dist/owletto/dist/assets/index-B04ON9Jv.js +0 -2
- package/dist/owletto/dist/assets/index-BOsK-K-2.js +0 -2
- package/dist/owletto/dist/assets/index-Bk1x7A67.js +0 -2
- package/dist/owletto/dist/assets/index-CFnx5huL.js +0 -2
- package/dist/owletto/dist/assets/index-Dbd1jusf.js +0 -2
- package/dist/owletto/dist/assets/index-DqZuqC08.js +0 -2
- package/dist/owletto/dist/assets/index-Dr-Xg_xO.js +0 -2
- package/dist/owletto/dist/assets/index-KPOvMVa-.js +0 -2
- package/dist/owletto/dist/assets/models-DnMb3n-Z.js +0 -2
- package/dist/owletto/dist/assets/public-shell-ready-7KYnww6e.js +0 -2
- package/dist/owletto/dist/assets/runs-panel-BO3h5RkS.js +0 -2
- package/dist/owletto/dist/assets/settings-4Ev-0iNL.js +0 -2
- package/dist/owletto/dist/assets/skills-D0X430X8.js +0 -2
- package/vendor/core/dist/contracts/worker/device-chat.d.ts +0 -4
- package/vendor/core/dist/contracts/worker/device-chat.js +0 -26
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure takeout parsing/formatting helpers for the LinkedIn data-export feeds
|
|
3
|
+
* (`linkedin.ts`, `linkedin-takeout.ts`).
|
|
4
|
+
*
|
|
5
|
+
* ISOLATE-SAFE BY CONTRACT: this module must never import `node:fs`,
|
|
6
|
+
* `node:path`, or any other builtin the isolate prelude does not provide.
|
|
7
|
+
* Connector bundles run in a V8 isolate and `assertIsolateEligible` rejects the
|
|
8
|
+
* WHOLE bundle over one surviving builtin `require`, so an `fs` import here
|
|
9
|
+
* would take down the live browser feeds of every connector that shares these
|
|
10
|
+
* helpers — the LinkedIn regression in Lobu#3392.
|
|
11
|
+
*
|
|
12
|
+
* `node:crypto` is fine: the guest prelude serves it from `global.crypto`
|
|
13
|
+
* (see ISOLATE_PRELUDE_PROVIDED_BUILTINS).
|
|
14
|
+
*
|
|
15
|
+
* The CSV reader is injected into the connector (see its constructor), never
|
|
16
|
+
* imported, so no filesystem code reaches the bundle.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
import type { EventEnvelope } from "@lobu/connector-sdk";
|
|
21
|
+
|
|
22
|
+
const DEFAULT_WATERMARK = "1970-01-01T00:00:00.000Z";
|
|
23
|
+
const DEFAULT_BATCH_SIZE = 1000;
|
|
24
|
+
|
|
25
|
+
export interface LocalTakeoutConfig {
|
|
26
|
+
takeout_dir?: string;
|
|
27
|
+
batch_size?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function batchSize(config: LocalTakeoutConfig): number {
|
|
31
|
+
return Math.max(1, Math.min(config.batch_size ?? DEFAULT_BATCH_SIZE, 5000));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function takeBatch<T extends EventEnvelope>(
|
|
35
|
+
events: T[],
|
|
36
|
+
watermark: string | undefined,
|
|
37
|
+
max: number
|
|
38
|
+
): T[] {
|
|
39
|
+
const after = watermark ?? DEFAULT_WATERMARK;
|
|
40
|
+
return events
|
|
41
|
+
.filter((event) => eventCursor(event) > after)
|
|
42
|
+
.sort((a, b) => eventCursor(a).localeCompare(eventCursor(b)))
|
|
43
|
+
.slice(0, max);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function maxEventCursor(
|
|
47
|
+
events: EventEnvelope[],
|
|
48
|
+
fallback?: string
|
|
49
|
+
): string {
|
|
50
|
+
return events.reduce(
|
|
51
|
+
(max, event) => (eventCursor(event) > max ? eventCursor(event) : max),
|
|
52
|
+
fallback ?? DEFAULT_WATERMARK
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Composite watermark: ISO timestamp + origin_id so events sharing a second
|
|
57
|
+
// still order deterministically. The delimiter is persisted inside the jsonb
|
|
58
|
+
// checkpoint column, so it must NOT be NUL: Postgres rejects NUL in jsonb
|
|
59
|
+
// (unsupported Unicode escape sequence). \u0001 (SOH) sorts before every
|
|
60
|
+
// printable char (preserving timestamp-first ordering) and is jsonb-safe.
|
|
61
|
+
const CURSOR_DELIMITER = "\u0001";
|
|
62
|
+
|
|
63
|
+
function eventCursor(event: EventEnvelope): string {
|
|
64
|
+
return `${event.occurred_at.toISOString()}${CURSOR_DELIMITER}${event.origin_id}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function stableId(
|
|
68
|
+
prefix: string,
|
|
69
|
+
parts: Array<string | number | undefined>
|
|
70
|
+
): string {
|
|
71
|
+
const hash = createHash("sha256")
|
|
72
|
+
.update(parts.map((part) => String(part ?? "")).join("\0"))
|
|
73
|
+
.digest("hex")
|
|
74
|
+
.slice(0, 32);
|
|
75
|
+
return `${prefix}_${hash}`;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function stripHtml(html: string): string {
|
|
79
|
+
return decodeHtml(htmlToPlainText(html))
|
|
80
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
81
|
+
.replace(/[ \t]{2,}/g, " ")
|
|
82
|
+
.trim();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function htmlToPlainText(html: string): string {
|
|
86
|
+
let output = "";
|
|
87
|
+
let index = 0;
|
|
88
|
+
const lowerHtml = html.toLowerCase();
|
|
89
|
+
|
|
90
|
+
while (index < html.length) {
|
|
91
|
+
const char = html[index];
|
|
92
|
+
if (char !== "<") {
|
|
93
|
+
output += char;
|
|
94
|
+
index += 1;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const tagEnd = html.indexOf(">", index + 1);
|
|
99
|
+
if (tagEnd === -1) {
|
|
100
|
+
output += char;
|
|
101
|
+
index += 1;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const tagName = readTagName(html.slice(index + 1, tagEnd));
|
|
106
|
+
if (tagName === "script" || tagName === "style") {
|
|
107
|
+
const closeTagStart = lowerHtml.indexOf(`</${tagName}`, tagEnd + 1);
|
|
108
|
+
if (closeTagStart === -1) {
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
const closeTagEnd = html.indexOf(">", closeTagStart + tagName.length + 2);
|
|
112
|
+
index = closeTagEnd === -1 ? html.length : closeTagEnd + 1;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (tagName === "br" || isBlockBreakTag(tagName)) {
|
|
117
|
+
output += "\n";
|
|
118
|
+
}
|
|
119
|
+
index = tagEnd + 1;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return output;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function readTagName(tag: string): string {
|
|
126
|
+
let index = tag[0] === "/" ? 1 : 0;
|
|
127
|
+
while (tag[index] === " " || tag[index] === "\t" || tag[index] === "\n") {
|
|
128
|
+
index += 1;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
let name = "";
|
|
132
|
+
while (index < tag.length) {
|
|
133
|
+
const char = tag[index]?.toLowerCase();
|
|
134
|
+
if (!char || !isAsciiNameChar(char)) {
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
name += char;
|
|
138
|
+
index += 1;
|
|
139
|
+
}
|
|
140
|
+
return name;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function isAsciiNameChar(char: string): boolean {
|
|
144
|
+
return (
|
|
145
|
+
(char >= "a" && char <= "z") || (char >= "0" && char <= "9") || char === "-"
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function isBlockBreakTag(tagName: string): boolean {
|
|
150
|
+
return (
|
|
151
|
+
tagName === "p" ||
|
|
152
|
+
tagName === "div" ||
|
|
153
|
+
tagName === "li" ||
|
|
154
|
+
tagName === "h1" ||
|
|
155
|
+
tagName === "h2" ||
|
|
156
|
+
tagName === "h3" ||
|
|
157
|
+
tagName === "h4" ||
|
|
158
|
+
tagName === "h5" ||
|
|
159
|
+
tagName === "h6"
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function decodeHtml(text: string): string {
|
|
164
|
+
return text.replace(
|
|
165
|
+
/&(nbsp|amp|quot|#039|apos|#\d+|#x[0-9a-f]+);/gi,
|
|
166
|
+
(entity, code) => {
|
|
167
|
+
const normalized = String(code).toLowerCase();
|
|
168
|
+
if (normalized === "nbsp") return " ";
|
|
169
|
+
if (normalized === "amp") return "&";
|
|
170
|
+
if (normalized === "quot") return '"';
|
|
171
|
+
if (normalized === "#039" || normalized === "apos") return "'";
|
|
172
|
+
if (normalized.startsWith("#x")) {
|
|
173
|
+
const codePoint = Number.parseInt(normalized.slice(2), 16);
|
|
174
|
+
return decodeHtmlCodePoint(codePoint, entity);
|
|
175
|
+
}
|
|
176
|
+
if (normalized.startsWith("#")) {
|
|
177
|
+
const codePoint = Number(normalized.slice(1));
|
|
178
|
+
return decodeHtmlCodePoint(codePoint, entity);
|
|
179
|
+
}
|
|
180
|
+
return entity;
|
|
181
|
+
}
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function decodeHtmlCodePoint(codePoint: number, fallback: string): string {
|
|
186
|
+
if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
|
|
187
|
+
return fallback;
|
|
188
|
+
}
|
|
189
|
+
if (codePoint === 60 || codePoint === 62) {
|
|
190
|
+
return fallback;
|
|
191
|
+
}
|
|
192
|
+
return String.fromCodePoint(codePoint);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function parseCsv(text: string): Record<string, string>[] {
|
|
196
|
+
const rows: string[][] = [];
|
|
197
|
+
let row: string[] = [];
|
|
198
|
+
let cell = "";
|
|
199
|
+
let inQuotes = false;
|
|
200
|
+
|
|
201
|
+
for (let i = 0; i < text.length; i++) {
|
|
202
|
+
const char = text[i];
|
|
203
|
+
const next = text[i + 1];
|
|
204
|
+
if (char === '"') {
|
|
205
|
+
if (inQuotes && next === '"') {
|
|
206
|
+
cell += '"';
|
|
207
|
+
i++;
|
|
208
|
+
} else {
|
|
209
|
+
inQuotes = !inQuotes;
|
|
210
|
+
}
|
|
211
|
+
} else if (char === "," && !inQuotes) {
|
|
212
|
+
row.push(cell);
|
|
213
|
+
cell = "";
|
|
214
|
+
} else if ((char === "\n" || char === "\r") && !inQuotes) {
|
|
215
|
+
if (char === "\r" && next === "\n") i++;
|
|
216
|
+
row.push(cell);
|
|
217
|
+
if (row.some((value) => value.trim().length > 0)) rows.push(row);
|
|
218
|
+
row = [];
|
|
219
|
+
cell = "";
|
|
220
|
+
} else {
|
|
221
|
+
cell += char;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
row.push(cell);
|
|
225
|
+
if (row.some((value) => value.trim().length > 0)) rows.push(row);
|
|
226
|
+
|
|
227
|
+
const headerIndex = rows.findIndex((candidate) => {
|
|
228
|
+
const values = candidate.map((value) => value.trim()).filter(Boolean);
|
|
229
|
+
return values.length > 1 && !values[0]?.startsWith("Notes:");
|
|
230
|
+
});
|
|
231
|
+
if (headerIndex < 0) return [];
|
|
232
|
+
const headers = rows[headerIndex].map((value) => value.trim());
|
|
233
|
+
return rows.slice(headerIndex + 1).map((values) => {
|
|
234
|
+
const record: Record<string, string> = {};
|
|
235
|
+
headers.forEach((header, index) => {
|
|
236
|
+
record[header] = values[index]?.trim() ?? "";
|
|
237
|
+
});
|
|
238
|
+
return record;
|
|
239
|
+
});
|
|
240
|
+
}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// Serialized into the paired browser. Read only the inbox's own loaded records
|
|
2
2
|
// and read loaders; never return provider clients, tickets, or arbitrary state.
|
|
3
3
|
export async function tiktokInboxPage(input) {
|
|
4
|
-
const fail = message => { throw new Error(`TikTok inbox: ${message}`); };
|
|
4
|
+
const fail = (message, reason) => { throw Object.assign(new Error(`TikTok inbox: ${message}`), reason ? { browser_interruption: reason } : {}); };
|
|
5
5
|
const visible = element => !!element?.getClientRects().length;
|
|
6
6
|
const account = () => {
|
|
7
7
|
if (location.origin !== "https://www.tiktok.com") fail("Wrong page origin");
|
|
8
|
-
if (visible(document.querySelector("#captcha-verify-container-main-page"))) fail("Verification required in the connected browser");
|
|
8
|
+
if (visible(document.querySelector("#captcha-verify-container-main-page"))) fail("Verification required in the connected browser", "challenge_required");
|
|
9
9
|
const raw = document.querySelector("#__UNIVERSAL_DATA_FOR_REHYDRATION__")?.textContent;
|
|
10
10
|
const user = raw && JSON.parse(raw).__DEFAULT_SCOPE__?.["webapp.app-context"]?.user;
|
|
11
|
-
if (typeof user?.uid !== "string" || !/^\d+$/.test(user.uid)) fail("Sign-in is required");
|
|
11
|
+
if (typeof user?.uid !== "string" || !/^\d+$/.test(user.uid)) fail("Sign-in is required", "login_required");
|
|
12
12
|
return user.uid;
|
|
13
13
|
};
|
|
14
14
|
const owner = account();
|
|
@@ -16,12 +16,12 @@ export async function tiktokInboxPage(input) {
|
|
|
16
16
|
const pause = () => new Promise(resolve => setTimeout(resolve, 100));
|
|
17
17
|
const wait = async read => {
|
|
18
18
|
while (Date.now() < deadline) {
|
|
19
|
-
if (account() !== owner) fail("Account changed during the read");
|
|
19
|
+
if (account() !== owner) fail("Account changed during the read", "account_mismatch");
|
|
20
20
|
const value = read();
|
|
21
21
|
if (value) return value;
|
|
22
22
|
await pause();
|
|
23
23
|
}
|
|
24
|
-
fail("Inbox did not finish loading; restart this page read");
|
|
24
|
+
fail("Inbox did not finish loading; restart this page read", "loading_failure");
|
|
25
25
|
};
|
|
26
26
|
const readWithinDeadline = async read => {
|
|
27
27
|
let outcome;
|
|
@@ -47,7 +47,7 @@ export async function tiktokInboxPage(input) {
|
|
|
47
47
|
const bytes = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(JSON.stringify(row)));
|
|
48
48
|
return { ...row, source_revision: [...new Uint8Array(bytes)].map(byte => byte.toString(16).padStart(2, "0")).join("") };
|
|
49
49
|
}));
|
|
50
|
-
if (account() !== owner) fail("Account changed during the read");
|
|
50
|
+
if (account() !== owner) fail("Account changed during the read", "account_mismatch");
|
|
51
51
|
return { rows: revised, hasMore: !!next, ...(next ? { nextCursor: JSON.stringify({ v: feed === "messages" ? 2 : 1, feed, account: owner, ...next }) } : {}) };
|
|
52
52
|
};
|
|
53
53
|
// React attaches the source record and loading state to these exact inbox
|
|
@@ -162,7 +162,7 @@ export async function tiktokInboxPage(input) {
|
|
|
162
162
|
return null;
|
|
163
163
|
};
|
|
164
164
|
const api = await wait(clientApi);
|
|
165
|
-
if (api.instance.ctx?.option?.userId !== owner) fail("Message client belongs to a different account");
|
|
165
|
+
if (api.instance.ctx?.option?.userId !== owner) fail("Message client belongs to a different account", "account_mismatch");
|
|
166
166
|
await wait(() => {
|
|
167
167
|
if (api.stateStore.isInitError || api.stateStore.isNotAvailable) fail("Message initialization failed");
|
|
168
168
|
return api.stateStore.isInitSucceeded && api.stateStore.isInitLoadDone;
|
|
@@ -175,7 +175,7 @@ export async function tiktokInboxPage(input) {
|
|
|
175
175
|
const snapshot = [...ids].sort();
|
|
176
176
|
const finish = async (rows, next) => {
|
|
177
177
|
const output = await result(rows, next);
|
|
178
|
-
if (api.instance.ctx?.option?.userId !== owner) fail("Message client changed accounts during the read");
|
|
178
|
+
if (api.instance.ctx?.option?.userId !== owner) fail("Message client changed accounts during the read", "account_mismatch");
|
|
179
179
|
const latest = api.instance.getConversationList();
|
|
180
180
|
if (!Array.isArray(latest) || JSON.stringify(latest.map(thread => thread.id).sort()) !== JSON.stringify(snapshot)) fail("Conversation set changed; restart the read, previous coverage is incomplete");
|
|
181
181
|
return output;
|
|
@@ -204,7 +204,7 @@ export async function tiktokInboxPage(input) {
|
|
|
204
204
|
};
|
|
205
205
|
for (let page = 0; page < 20; page++) {
|
|
206
206
|
await wait(() => !thread.isPullingMessages);
|
|
207
|
-
if (api.instance.ctx?.option?.userId !== owner) fail("Message client changed accounts during the read");
|
|
207
|
+
if (api.instance.ctx?.option?.userId !== owner) fail("Message client changed accounts during the read", "account_mismatch");
|
|
208
208
|
const messages = thread.getMessageList().map(project).sort((a,b) => b.occurred_at.localeCompare(a.occurred_at) || b.id.localeCompare(a.id));
|
|
209
209
|
const start = cursor.after ? messages.findIndex(message => message.id === cursor.after) + 1 : 0;
|
|
210
210
|
if ((cursor.after && !start) || start === messages.length && thread.hasMoreMessages) {
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// Self-contained browser adapter. Serialized into the user's paired browser;
|
|
2
2
|
// keep every dependency inside this function, as in the WhatsApp page adapter.
|
|
3
3
|
export async function tiktokPage(command, args = {}) {
|
|
4
|
+
const interrupt = (reason, message) => { throw Object.assign(new Error(message), { browser_interruption: reason }); };
|
|
4
5
|
const visible = (e) => !!e && e.getClientRects().length > 0;
|
|
5
6
|
const assertNoChallenge = () => {
|
|
6
7
|
if (visible(document.querySelector("#captcha-verify-container-main-page"))) {
|
|
7
|
-
|
|
8
|
+
interrupt("challenge_required", "TikTok verification required; complete the challenge in the connected browser");
|
|
8
9
|
}
|
|
9
10
|
};
|
|
10
11
|
assertNoChallenge();
|
|
@@ -19,27 +20,36 @@ export async function tiktokPage(command, args = {}) {
|
|
|
19
20
|
};
|
|
20
21
|
const wait = async (read) => {
|
|
21
22
|
for (let i = 0; i < 40; i++) {
|
|
23
|
+
assertNoChallenge();
|
|
22
24
|
const value = read();
|
|
23
25
|
if (value) return value;
|
|
24
26
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
25
27
|
}
|
|
26
|
-
|
|
28
|
+
interrupt("loading_failure", "TikTok page did not expose the requested content");
|
|
27
29
|
};
|
|
28
30
|
if (command === "identity") {
|
|
29
|
-
try { return await wait(identity); } catch {
|
|
31
|
+
try { return await wait(identity); } catch (error) {
|
|
32
|
+
if (error.browser_interruption !== "loading_failure") throw error;
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
30
35
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
const owner = identity()?.accountId;
|
|
37
|
+
if (!owner) interrupt("login_required", "TikTok sign-in is required");
|
|
38
|
+
if (command === "read_failure") interrupt("loading_failure", "TikTok page did not finish loading the requested content");
|
|
39
|
+
const assertTimeline = (path) => {
|
|
40
|
+
if (location.origin !== "https://www.tiktok.com" || location.pathname !== path) throw new Error("TikTok timeline page changed");
|
|
34
41
|
assertNoChallenge();
|
|
35
|
-
if (!identity())
|
|
42
|
+
if (!identity()) interrupt("login_required", "TikTok sign-in is required");
|
|
43
|
+
if (identity().accountId !== owner) interrupt("account_mismatch", "TikTok account changed during the read");
|
|
36
44
|
};
|
|
37
|
-
|
|
38
|
-
|
|
45
|
+
const assertFollowing = () => assertTimeline("/following");
|
|
46
|
+
if (["following_status", "following_wait", "for_you_status", "for_you_wait"].includes(command)) {
|
|
47
|
+
const path = command.startsWith("following_") ? "/following" : "/";
|
|
48
|
+
assertTimeline(path);
|
|
39
49
|
document.querySelectorAll("video,audio").forEach(media => { media.muted = true; });
|
|
40
|
-
if (command
|
|
50
|
+
if (command.endsWith("_wait")) {
|
|
41
51
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
42
|
-
|
|
52
|
+
assertTimeline(path);
|
|
43
53
|
}
|
|
44
54
|
return { ready: true };
|
|
45
55
|
}
|
|
@@ -94,10 +104,6 @@ export async function tiktokPage(command, args = {}) {
|
|
|
94
104
|
if (seen.size < args.limit && hasMore) throw new Error(`TikTok Following read incomplete: collected ${seen.size} of ${args.limit} requested posts; retry the read`);
|
|
95
105
|
return { items: [...items.values()] };
|
|
96
106
|
}
|
|
97
|
-
if (command === "snapshot") {
|
|
98
|
-
const d = scope();
|
|
99
|
-
return { items: (d["webapp.updated-items"] || []).map(project) };
|
|
100
|
-
}
|
|
101
107
|
if (command === "post_links") {
|
|
102
108
|
// Hidden preload links appear before search results render. Their presence
|
|
103
109
|
// satisfies wait_for_selector but cannot establish readable coverage.
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
|
+
assertInboxSourceRead,
|
|
3
|
+
assertPositiveReadLimit,
|
|
4
|
+
assertSnapshotSourceRead,
|
|
2
5
|
ConnectorRuntime,
|
|
3
6
|
requireBrowser,
|
|
4
7
|
type ActionContext,
|
|
@@ -26,10 +29,15 @@ export function normalizeTikTokPostUrl(value: unknown): string {
|
|
|
26
29
|
return `${ORIGIN}${url.pathname}`;
|
|
27
30
|
}
|
|
28
31
|
|
|
32
|
+
function hasPostIdentity(item: RecordValue): boolean {
|
|
33
|
+
const author = object(item.author);
|
|
34
|
+
return typeof item.id === "string" && /^[0-9]+$/.test(item.id) && typeof author.uniqueId === "string" && HANDLE.test(author.uniqueId);
|
|
35
|
+
}
|
|
36
|
+
|
|
29
37
|
export function normalizeTikTokPost(value: unknown): RecordValue {
|
|
30
38
|
const item = object(value);
|
|
31
39
|
const author = object(item.author);
|
|
32
|
-
if (
|
|
40
|
+
if (!hasPostIdentity(item)) {
|
|
33
41
|
throw new Error("TikTok returned a post without a valid source identity");
|
|
34
42
|
}
|
|
35
43
|
const video = object(item.video);
|
|
@@ -49,29 +57,49 @@ export function normalizeTikTokPost(value: unknown): RecordValue {
|
|
|
49
57
|
};
|
|
50
58
|
}
|
|
51
59
|
|
|
60
|
+
// Keep structured page failures in a successful CDP value. CDP exceptions only
|
|
61
|
+
// preserve prose, which cannot safely distinguish a challenge from a timeout.
|
|
62
|
+
function pageExpression(source: string): string {
|
|
63
|
+
return `Promise.resolve().then(() => ${source}).catch(error => { if (!error.browser_interruption) throw error; return { __lobu_browser_interruption: { reason: error.browser_interruption, message: error.message } }; })`;
|
|
64
|
+
}
|
|
65
|
+
|
|
52
66
|
function expression(command: string, args: RecordValue = {}): string {
|
|
53
|
-
return `(${tiktokPage.toString()})(${JSON.stringify(command)},${JSON.stringify(args)})
|
|
67
|
+
return pageExpression(`(${tiktokPage.toString()})(${JSON.stringify(command)},${JSON.stringify(args)})`);
|
|
54
68
|
}
|
|
55
69
|
|
|
56
70
|
async function page(browser: ChromeActionDispatcher, tabId: number, command: string, args: RecordValue = {}): Promise<RecordValue> {
|
|
57
71
|
const output = await browser.dispatch("evaluate", { tab_id: tabId, expression: expression(command, args), allowed_origins: ORIGINS });
|
|
72
|
+
// Only the source-read bridge throws a typed interruption; operation and
|
|
73
|
+
// worker dispatchers return the marker as a completed value.
|
|
74
|
+
const interruption = object(object(output.value).__lobu_browser_interruption);
|
|
75
|
+
if (interruption.reason) throw new Error(String(interruption.message));
|
|
58
76
|
if (!output.value || typeof output.value !== "object") throw new Error("TikTok page returned no data");
|
|
59
77
|
return object(output.value);
|
|
60
78
|
}
|
|
61
79
|
|
|
62
80
|
async function scratch<T>(browser: ChromeActionDispatcher, run: (tabId: number) => Promise<T>): Promise<T> {
|
|
63
|
-
const nav = await browser.dispatch("navigate", { url: "about:blank", new_tab: true, allowed_origins: ORIGINS });
|
|
81
|
+
const nav = await browser.dispatch("navigate", { url: "about:blank", new_tab: true, retain_on_interruption: true, allowed_origins: ORIGINS });
|
|
64
82
|
if (typeof nav.tab_id !== "number") throw new Error("TikTok scratch tab was not created");
|
|
65
83
|
try { return await run(nav.tab_id); }
|
|
66
84
|
finally { await browser.dispatch("close_tab", { tab_id: nav.tab_id, allowed_origins: ORIGINS }); }
|
|
67
85
|
}
|
|
68
86
|
|
|
87
|
+
async function waitFor(browser: ChromeActionDispatcher, tabId: number, selector: string): Promise<void> {
|
|
88
|
+
try { await browser.dispatch("wait_for_selector", { tab_id: tabId, selector, timeout_ms: 15000, allowed_origins: ORIGINS }); }
|
|
89
|
+
catch (error) {
|
|
90
|
+
// Inspect the page while it is still owned. A selector timeout alone is
|
|
91
|
+
// neither verification evidence nor permission for a recovery handoff.
|
|
92
|
+
await page(browser, tabId, "read_failure");
|
|
93
|
+
throw error;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
69
97
|
async function navigate(browser: ChromeActionDispatcher, tabId: number, url: string, focus = false): Promise<void> {
|
|
70
98
|
await browser.dispatch("navigate", { tab_id: tabId, url, allowed_origins: ORIGINS });
|
|
71
99
|
// TikTok's video page can remain blank in a fresh background tab. Its own
|
|
72
100
|
// controls appeared immediately after focus in the live browser probe.
|
|
73
101
|
if (focus) await browser.dispatch("focus_tab", { tab_id: tabId, draw_attention: false, allowed_origins: ORIGINS });
|
|
74
|
-
await browser
|
|
102
|
+
await waitFor(browser, tabId, '[data-e2e="nav-profile"]');
|
|
75
103
|
}
|
|
76
104
|
|
|
77
105
|
async function sourcePost(browser: ChromeActionDispatcher, tabId: number, url: string): Promise<RecordValue> {
|
|
@@ -83,7 +111,7 @@ async function sourcePost(browser: ChromeActionDispatcher, tabId: number, url: s
|
|
|
83
111
|
if (typeof start.session_id !== "string") throw new Error("TikTok photo capture did not start");
|
|
84
112
|
try {
|
|
85
113
|
await navigate(browser, tabId, url);
|
|
86
|
-
await browser
|
|
114
|
+
await waitFor(browser, tabId, '[data-e2e="recommend-list-item-container"] .swiper');
|
|
87
115
|
const data = await browser.dispatch("network_intercept_drain", { session_id: start.session_id, allowed_origins: ORIGINS });
|
|
88
116
|
for (const response of Array.isArray(data.responses) ? data.responses : []) {
|
|
89
117
|
const r = object(response);
|
|
@@ -98,25 +126,31 @@ async function sourcePost(browser: ChromeActionDispatcher, tabId: number, url: s
|
|
|
98
126
|
}
|
|
99
127
|
}
|
|
100
128
|
|
|
101
|
-
async function
|
|
129
|
+
async function timelinePosts(browser: ChromeActionDispatcher, tabId: number, sessionId: string, limit: number, feedKey: "for_you" | "following"): Promise<RecordValue[]> {
|
|
130
|
+
const endpoint = feedKey === "following" ? "following" : "recommend";
|
|
131
|
+
const label = feedKey === "following" ? "Following" : "For You";
|
|
102
132
|
const unique = new Map<unknown, RecordValue>();
|
|
103
133
|
let continuation: RecordValue | undefined;
|
|
104
134
|
for (let poll = 0; poll < 5; poll++) {
|
|
105
|
-
await page(browser, tabId, poll ? "
|
|
135
|
+
await page(browser, tabId, `${feedKey}_${poll ? "wait" : "status"}`);
|
|
106
136
|
const data = await browser.dispatch("network_intercept_drain", { session_id: sessionId, allowed_origins: ORIGINS });
|
|
107
|
-
if (data.missing || !Array.isArray(data.responses)) throw new Error(
|
|
137
|
+
if (data.missing || !Array.isArray(data.responses)) throw new Error(`TikTok ${label} capture is unavailable`);
|
|
108
138
|
for (const response of data.responses) {
|
|
109
139
|
const r = object(response);
|
|
110
140
|
const url = new URL(String(r.url));
|
|
111
|
-
if (url.origin !== ORIGIN || url.pathname !==
|
|
112
|
-
if (r.status !== 200 || r.truncated || r.base64_encoded || typeof r.body !== "string") throw new Error(
|
|
141
|
+
if (url.origin !== ORIGIN || url.pathname !== `/api/${endpoint}/item_list/` || url.username || url.password) throw new Error(`TikTok ${label} capture has an unexpected URL`);
|
|
142
|
+
if (r.status !== 200 || r.truncated || r.base64_encoded || typeof r.body !== "string") throw new Error(`TikTok ${label} response is unavailable or truncated`);
|
|
113
143
|
const json = object(JSON.parse(r.body));
|
|
114
|
-
if (json.status_code !== 0 || (json.statusCode !== undefined && json.statusCode !== 0) || !Array.isArray(json.itemList) || typeof json.hasMore !== "boolean") throw new Error(
|
|
144
|
+
if (json.status_code !== 0 || (json.statusCode !== undefined && json.statusCode !== 0) || !Array.isArray(json.itemList) || typeof json.hasMore !== "boolean") throw new Error(`TikTok did not return a successful ${label} feed`);
|
|
115
145
|
for (const item of json.itemList) {
|
|
146
|
+
const record = object(item);
|
|
147
|
+
// Some recommendation ads use an advertiser label in uniqueId,
|
|
148
|
+
// not a creator handle. They cannot produce a canonical post URL.
|
|
149
|
+
if (feedKey === "for_you" && record.isAd === true && !hasPostIdentity(record)) continue;
|
|
116
150
|
const row = normalizeTikTokPost(item);
|
|
117
151
|
unique.set(row.origin_id, row);
|
|
118
152
|
}
|
|
119
|
-
continuation = json.hasMore ? { url: url.href, cursor: json.cursor, level: json.level } : undefined;
|
|
153
|
+
continuation = feedKey === "following" && json.hasMore ? { url: url.href, cursor: json.cursor, level: json.level } : undefined;
|
|
120
154
|
}
|
|
121
155
|
if (data.responses.length) break;
|
|
122
156
|
}
|
|
@@ -131,14 +165,14 @@ async function followingPosts(browser: ChromeActionDispatcher, tabId: number, se
|
|
|
131
165
|
}
|
|
132
166
|
}
|
|
133
167
|
// Do not return a successful partial read when a late challenge appeared.
|
|
134
|
-
await page(browser, tabId,
|
|
168
|
+
await page(browser, tabId, `${feedKey}_status`);
|
|
135
169
|
return [...unique.values()];
|
|
136
170
|
}
|
|
137
171
|
|
|
138
172
|
async function readTikTokSnapshot(ctx: FeedReadContext): Promise<FeedReadResult> {
|
|
139
|
-
|
|
173
|
+
assertSnapshotSourceRead(ctx, "TikTok");
|
|
140
174
|
if (ctx.query && ctx.feedKey !== "search") throw new Error("Use the TikTok search feed for queries");
|
|
141
|
-
|
|
175
|
+
assertPositiveReadLimit(ctx);
|
|
142
176
|
const limit = Math.min(ctx.limit ?? 10, 30);
|
|
143
177
|
let url = ORIGIN;
|
|
144
178
|
if (ctx.feedKey === "following") url += "/following";
|
|
@@ -151,28 +185,26 @@ async function readTikTokSnapshot(ctx: FeedReadContext): Promise<FeedReadResult>
|
|
|
151
185
|
return scratch(browser, async tabId => {
|
|
152
186
|
let sessionId: string | undefined;
|
|
153
187
|
try {
|
|
154
|
-
if (ctx.feedKey
|
|
155
|
-
const
|
|
188
|
+
if (ctx.feedKey !== "search") {
|
|
189
|
+
const endpoint = ctx.feedKey === "following" ? "following" : "recommend";
|
|
190
|
+
const start = await browser.dispatch("network_intercept_start", { tab_id: tabId, patterns: [`${ORIGIN}/api/${endpoint}/item_list/**`], max_body_bytes: 2097152, max_buffer_responses: 10, allowed_origins: ORIGINS });
|
|
156
191
|
if (typeof start.session_id !== "string") throw new Error("TikTok feed capture did not start");
|
|
157
192
|
sessionId = start.session_id;
|
|
158
193
|
}
|
|
159
194
|
// Snapshot/search results also lazy-load only in a visible tab.
|
|
160
195
|
await navigate(browser, tabId, url, true);
|
|
161
196
|
let rows: RecordValue[] = [];
|
|
162
|
-
if (ctx.feedKey === "following") {
|
|
163
|
-
await browser
|
|
164
|
-
rows = await
|
|
165
|
-
} else
|
|
166
|
-
await browser
|
|
197
|
+
if (ctx.feedKey === "following" || ctx.feedKey === "for_you") {
|
|
198
|
+
await waitFor(browser, tabId, '[data-e2e="recommend-list-item-container"]');
|
|
199
|
+
rows = await timelinePosts(browser, tabId, sessionId!, limit, ctx.feedKey);
|
|
200
|
+
} else {
|
|
201
|
+
await waitFor(browser, tabId, 'a[href*="/video/"],a[href*="/photo/"]');
|
|
167
202
|
const data = await page(browser, tabId, "post_links");
|
|
168
203
|
rows = (Array.isArray(data.rows) ? data.rows : []).map(value => {
|
|
169
204
|
const r = object(value);
|
|
170
205
|
const sourceUrl = normalizeTikTokPostUrl(r.url);
|
|
171
206
|
return { origin_id: sourceUrl.split("/").pop(), source_url: sourceUrl, text: r.text, media: { type: sourceUrl.includes("/photo/") ? "photos" : "video" } };
|
|
172
207
|
});
|
|
173
|
-
} else {
|
|
174
|
-
const data = await page(browser, tabId, "snapshot");
|
|
175
|
-
rows = (Array.isArray(data.items) ? data.items : []).map(normalizeTikTokPost);
|
|
176
208
|
}
|
|
177
209
|
if (!rows.length) throw new Error("TikTok returned no verifiable posts; this is not proof of an empty feed");
|
|
178
210
|
const unique = [...new Map(rows.map(row => [row.origin_id, row])).values()];
|
|
@@ -186,18 +218,19 @@ async function readTikTokSnapshot(ctx: FeedReadContext): Promise<FeedReadResult>
|
|
|
186
218
|
}
|
|
187
219
|
|
|
188
220
|
async function readTikTokInbox(ctx: FeedReadContext): Promise<FeedReadResult> {
|
|
189
|
-
|
|
190
|
-
|
|
221
|
+
assertInboxSourceRead(ctx, "TikTok");
|
|
222
|
+
assertPositiveReadLimit(ctx);
|
|
191
223
|
const browser = requireBrowser(ctx);
|
|
192
224
|
return scratch(browser, async tabId => {
|
|
193
225
|
await navigate(browser, tabId, ORIGIN);
|
|
194
226
|
const result = await browser.dispatch("evaluate", {
|
|
195
227
|
tab_id: tabId,
|
|
196
|
-
expression: `(${tiktokInboxPage.toString()})(${JSON.stringify({ feed: ctx.feedKey, limit: Math.min(ctx.limit ?? 20, 100), cursor: ctx.cursor })})
|
|
228
|
+
expression: pageExpression(`(${tiktokInboxPage.toString()})(${JSON.stringify({ feed: ctx.feedKey, limit: Math.min(ctx.limit ?? 20, 100), cursor: ctx.cursor })})`),
|
|
197
229
|
await_promise: true,
|
|
198
230
|
allowed_origins: ORIGINS,
|
|
199
231
|
});
|
|
200
232
|
const data = object(result.value);
|
|
233
|
+
if (data.__lobu_browser_interruption) throw new Error(String(object(data.__lobu_browser_interruption).message));
|
|
201
234
|
if (!Array.isArray(data.rows) || typeof data.hasMore !== "boolean" || (data.hasMore && typeof data.nextCursor !== "string")) throw new Error("TikTok inbox returned no verifiable page");
|
|
202
235
|
return data as unknown as FeedReadResult;
|
|
203
236
|
});
|
|
@@ -205,14 +238,14 @@ async function readTikTokInbox(ctx: FeedReadContext): Promise<FeedReadResult> {
|
|
|
205
238
|
|
|
206
239
|
export default class TikTokConnector extends ConnectorRuntime {
|
|
207
240
|
readonly definition: RuntimeConnectorDefinition = {
|
|
208
|
-
key: "tiktok.web", name: "TikTok", version: "1.1.
|
|
241
|
+
key: "tiktok.web", name: "TikTok", version: "1.1.9", faviconDomain: "tiktok.com",
|
|
209
242
|
description: "Read TikTok through your signed-in paired browser, inspect post visuals, set likes, and prepare comments for human submission. Automations decide what to read and act on.",
|
|
210
243
|
browser: { origins: ORIGINS, authMethods: ["browser"], accountProbe: { url: ORIGIN, expression: expression("identity") } },
|
|
211
244
|
authSchema: { methods: [{ type: "browser", mode: "live" }] },
|
|
212
245
|
feeds: {
|
|
213
246
|
messages: { key: "messages", name: "Messages", description: "Read a recent page per conversation before older history, with sender, direction, source IDs, evidence revisions and timestamps. Cursors cover the initial conversation set; a changed set reports incomplete coverage and needs a fresh scan. Unsupported media is identified, not transcribed. No message is sent.", read: readTikTokInbox },
|
|
214
247
|
activity: { key: "activity", name: "Activity", description: "Read loaded TikTok activity including likes, comments, mentions and followers, with source IDs and links. Follow the returned cursor for older entries. Opening the activity panel may clear its unread badge.", read: readTikTokInbox },
|
|
215
|
-
for_you: { key: "for_you", name: "For You", description: "Initial personalized timeline snapshot. No history coverage or continuation; repeat reads can overlap.", read: readTikTokSnapshot },
|
|
248
|
+
for_you: { key: "for_you", name: "For You", description: "Initial personalized timeline snapshot. Ad units without canonical post identity are excluded. No history coverage or continuation; repeat reads can overlap.", read: readTikTokSnapshot },
|
|
216
249
|
following: { key: "following", name: "Following", description: "Read up to 30 distinct Following posts using TikTok's source continuation. Returns fewer only at the source end; stalled or budget-limited reads report an error. No complete history coverage or caller continuation; repeat reads can overlap.", read: readTikTokSnapshot },
|
|
217
250
|
search: { key: "search", name: "Search", description: "Browser search results for videos and photo posts. Use query when reading. Initial snapshot without continuation.", read: readTikTokSnapshot, configSchema: { type: "object", properties: { query: { type: "string", maxLength: 200 } }, additionalProperties: false } },
|
|
218
251
|
},
|