@scrapa/sdk 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -17
- package/dist/client.js +2 -2
- package/dist/client.js.map +1 -1
- package/dist/errors.js +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +66 -10
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +24 -9
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +77 -45
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +89 -43
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +250 -98
- package/dist/generated/params.d.ts.map +1 -1
- package/dist/generated/retry.d.ts +2 -2
- package/dist/generated/retry.js +3 -3
- package/dist/generated/transport.js +2 -2
- package/package.json +2 -2
- package/src/client.ts +2 -2
- package/src/errors.ts +1 -1
- package/src/generated/contract.ts +81 -9
- package/src/generated/operations.ts +109 -43
- package/src/generated/params.ts +252 -98
- package/src/generated/retry.ts +3 -3
- package/src/generated/transport.ts +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,
|
|
1
|
+
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;0FAMsF;IACtF,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;yDAGqD;IACrD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;gGAE4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;2FACuF;IACvF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;0FAMsF;IACtF,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;yDAGqD;IACrD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;gGAE4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;2FACuF;IACvF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,6BAA6B;AAC7B,MAAM,WAAW,aAAa;IAC1B,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,mGAAmG;IACnG,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;wBACoB;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;2EACuE;IACvE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;gFAC4E;IAC5E,OAAO,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,2FAA2F;IAC3F,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC3B,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,mGAAmG;IACnG,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;wBACoB;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;2EACuE;IACvE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;gFAC4E;IAC5E,OAAO,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,2FAA2F;IAC3F,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;qDACiD;IACjD,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;0FAMsF;IACtF,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;yDAGqD;IACrD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;gGAE4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;2FACuF;IACvF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;0FAMsF;IACtF,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;yDAGqD;IACrD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;gGAE4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;2FACuF;IACvF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
|
|
@@ -36,9 +36,9 @@ export interface RetryOptions {
|
|
|
36
36
|
/**
|
|
37
37
|
* Should this failure be retried?
|
|
38
38
|
*
|
|
39
|
-
* if (attempt < 3 &&
|
|
39
|
+
* if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
|
|
40
40
|
*/
|
|
41
|
-
export declare function
|
|
41
|
+
export declare function isRetryable(error: unknown, options: RetryOptions): boolean;
|
|
42
42
|
/**
|
|
43
43
|
* How long before retry number `attemptIndex` (0 for the first): the server's stated wait when
|
|
44
44
|
* there is one, an exponential backoff otherwise, and never less than the backoff. A
|
package/dist/generated/retry.js
CHANGED
|
@@ -9,9 +9,9 @@ const DEFAULT_MAX_WAIT_SECS = 10;
|
|
|
9
9
|
/**
|
|
10
10
|
* Should this failure be retried?
|
|
11
11
|
*
|
|
12
|
-
* if (attempt < 3 &&
|
|
12
|
+
* if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
|
|
13
13
|
*/
|
|
14
|
-
export function
|
|
14
|
+
export function isRetryable(error, options) {
|
|
15
15
|
const { repeatable, durableCodes = [], maxWaitSecs = DEFAULT_MAX_WAIT_SECS } = options;
|
|
16
16
|
const status = statusOf(error);
|
|
17
17
|
// No answer at all: nothing about it is a refusal, but a write may have landed before the
|
|
@@ -104,7 +104,7 @@ export function parseRetryAfter(value) {
|
|
|
104
104
|
* frees when another job ends. Present-and-`null` is the whole test; an absent key said nothing.
|
|
105
105
|
*
|
|
106
106
|
* Not exported. It is not "is this refusal durable?": it knows nothing of `durableCodes` or of a
|
|
107
|
-
* wait too long to hold, so a screen branching on it would disagree with `
|
|
107
|
+
* wait too long to hold, so a screen branching on it would disagree with `isRetryable`.
|
|
108
108
|
*/
|
|
109
109
|
function waitingNeverHelps(error) {
|
|
110
110
|
const details = field(error, "details");
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* - **The header is read first, in both its forms.** Two copies read the body's wait before the
|
|
24
24
|
* `Retry-After` header, and read the header only as seconds, so an HTTP date was no wait at all.
|
|
25
25
|
*/
|
|
26
|
-
import { parseRetryAfter, retryDelayMs,
|
|
26
|
+
import { parseRetryAfter, retryDelayMs, isRetryable } from "./retry.js";
|
|
27
27
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
28
28
|
const DEFAULT_MAX_RETRIES = 2;
|
|
29
29
|
/**
|
|
@@ -75,7 +75,7 @@ export async function send(transport, spec, params = {}, opts = {}) {
|
|
|
75
75
|
retryAfterSecs: failure.retryAfterSecs,
|
|
76
76
|
};
|
|
77
77
|
const again = attempt < maxRetries &&
|
|
78
|
-
|
|
78
|
+
isRetryable(answer, { repeatable, durableCodes: transport.durableCodes });
|
|
79
79
|
if (!again)
|
|
80
80
|
throw transport.error(failure);
|
|
81
81
|
await sleep(retryDelayMs(attempt, answer), signal);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@scrapa/sdk",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Official TypeScript client for scrapa.app
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Official TypeScript client for scrapa.app. Scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://scrapa.app",
|
|
7
7
|
"keywords": [
|
package/src/client.ts
CHANGED
|
@@ -116,11 +116,11 @@ export class Scrapa extends GeneratedOperations {
|
|
|
116
116
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
117
117
|
if (!opts.apiKey)
|
|
118
118
|
throw new TypeError(
|
|
119
|
-
"scrapa: an API key is required
|
|
119
|
+
"scrapa: an API key is required. Create one at https://console.scrapa.app/keys",
|
|
120
120
|
);
|
|
121
121
|
if (typeof (opts.fetch ?? globalThis.fetch) !== "function")
|
|
122
122
|
throw new TypeError(
|
|
123
|
-
"scrapa: no global fetch
|
|
123
|
+
"scrapa: no global fetch. Use Node 20+, or pass `fetch` in the options.",
|
|
124
124
|
);
|
|
125
125
|
const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
|
|
126
126
|
const host = new URL(baseUrl).host;
|
package/src/errors.ts
CHANGED
|
@@ -125,7 +125,7 @@ export function scrapaError(failure: Failure, host: string): ScrapaError {
|
|
|
125
125
|
status: 0,
|
|
126
126
|
mayHaveRun: method !== "GET",
|
|
127
127
|
message: failure.timedOut
|
|
128
|
-
? `scrapa: no answer within ${failure.timeoutMs}ms
|
|
128
|
+
? `scrapa: no answer within ${failure.timeoutMs}ms. The page may still have been read and charged.`
|
|
129
129
|
: `scrapa: could not reach ${host}.`,
|
|
130
130
|
cause: failure.cause,
|
|
131
131
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// GENERATED FILE — do not edit. Run `bun run sdk:gen`.
|
|
2
|
-
// Source: packages/shared/src/http/index.ts, packages/shared/src/locale/index.ts, packages/shared/src/enums/index.ts, packages/shared/src/billing/catalog.ts, packages/shared/src/dto/account.ts, packages/shared/src/dto/scrape.ts, packages/shared/src/dto/batch.ts, packages/shared/src/dto/search.ts, packages/shared/src/dto/sessions.ts, packages/shared/src/dto/usage.ts
|
|
2
|
+
// Source: packages/shared/src/http/index.ts, packages/shared/src/locale/index.ts, packages/shared/src/enums/index.ts, packages/shared/src/billing/catalog.ts, packages/shared/src/dto/account.ts, packages/shared/src/dto/scrape.ts, packages/shared/src/dto/batch.ts, packages/shared/src/dto/map.ts, packages/shared/src/dto/search.ts, packages/shared/src/dto/sessions.ts, packages/shared/src/dto/usage.ts
|
|
3
3
|
//
|
|
4
4
|
// The shapes the API speaks, lifted from the server's own contract.
|
|
5
5
|
|
|
@@ -11,8 +11,9 @@
|
|
|
11
11
|
* locale catalogs, exhaustiveness checks).
|
|
12
12
|
*
|
|
13
13
|
* THE WHOSE-FAULT VOCABULARY (DESIGN.md) is the organising principle: a caller's next move
|
|
14
|
-
* depends entirely on who said no, so the code says. `UPSTREAM_BLOCKED`
|
|
15
|
-
*
|
|
14
|
+
* depends entirely on who said no, so the code says. `UPSTREAM_BLOCKED` is "the site said no"
|
|
15
|
+
* (retry differently — another engine, pool or country), and `NAV_TIMEOUT` shares its hue under
|
|
16
|
+
* its own word, "time ran out" (give it more time; see `OWN_WORD`); `SSRF_DENIED` /
|
|
16
17
|
* `AUP_DENIED` are "our policy said no" (retrying cannot help, change the URL); a 5xx is
|
|
17
18
|
* "we broke it" (retry as-is, already refunded).
|
|
18
19
|
*/
|
|
@@ -44,6 +45,15 @@ export const ERROR_CODES = [
|
|
|
44
45
|
// or did not answer, or the managed pool has no exit in the place you asked for. Free,
|
|
45
46
|
// and no colour — the fix is in the request (the proxy, the credentials, a wider place).
|
|
46
47
|
"PROXY_REFUSED", // 422 — details {reason: "no_exit" | "refused" | "unreachable", place?, status?}
|
|
48
|
+
// The site sent a file bigger than `raw` hands back whole. Not 413, which tells an HTTP
|
|
49
|
+
// client its OWN request was too big. Free, and no colour, for PROXY_REFUSED's reason:
|
|
50
|
+
// the fix is in the request (another format, or download it yourself), not another road.
|
|
51
|
+
"FILE_TOO_LARGE", // 422 — details {limit}, in bytes
|
|
52
|
+
// A PDF or a Word file arrived whole, and reading it into text took longer than the time
|
|
53
|
+
// the call had left. Free, and no colour: the site answered in full, so another engine,
|
|
54
|
+
// pool or country reads the same file just as slowly. The fix is more `timeoutMs`, or
|
|
55
|
+
// "raw" and reading it yourself.
|
|
56
|
+
"DOCUMENT_TIMEOUT", // 422: details {budgetMs}, the call's timeout
|
|
47
57
|
"RATE_LIMIT_EXCEEDED", // 429 — the per-minute burst bucket; Retry-After + details {limit}
|
|
48
58
|
"QUOTA_EXCEEDED", // 429 — a durable bound that WAITING clears; details {scope, limit, resetAt}
|
|
49
59
|
"INTERNAL_ERROR", // 500 — we broke it; the body is always generic, the cause stays server-side
|
|
@@ -71,6 +81,8 @@ export const ERROR_STATUS = {
|
|
|
71
81
|
UPSTREAM_BLOCKED: 422,
|
|
72
82
|
NAV_TIMEOUT: 422,
|
|
73
83
|
PROXY_REFUSED: 422,
|
|
84
|
+
FILE_TOO_LARGE: 422,
|
|
85
|
+
DOCUMENT_TIMEOUT: 422,
|
|
74
86
|
RATE_LIMIT_EXCEEDED: 429,
|
|
75
87
|
QUOTA_EXCEEDED: 429,
|
|
76
88
|
INTERNAL_ERROR: 500,
|
|
@@ -79,10 +91,11 @@ export const ERROR_STATUS = {
|
|
|
79
91
|
} as const satisfies Record<ErrorCode, number>;
|
|
80
92
|
|
|
81
93
|
/**
|
|
82
|
-
* The three no-charge outcomes, as DESIGN.md names them.
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
94
|
+
* The three no-charge outcomes, as DESIGN.md names them. Each code here maps to exactly one
|
|
95
|
+
* — this is the map the console reads to pick the hue and the word, and the SDK reads to
|
|
96
|
+
* tell a caller what to do next. A code absent from here gets no colour: the caller's own
|
|
97
|
+
* mistake (a bad body, a missing key), or a free refusal whose fix is in the request rather
|
|
98
|
+
* than the site (PROXY_REFUSED, FILE_TOO_LARGE, DOCUMENT_TIMEOUT).
|
|
86
99
|
*
|
|
87
100
|
* blocked → the site said no → try another engine, pool or country
|
|
88
101
|
* denied → our policy said no → change the URL; retrying cannot help
|
|
@@ -180,7 +193,8 @@ export type ResolvedEngine = Exclude<Engine, "auto">;
|
|
|
180
193
|
/** What a scrape can hand back. `markdown` is the default because it is what an LLM, a
|
|
181
194
|
* notebook and a human all read; `html` is the page as fetched; `text` is prose with the
|
|
182
195
|
* markup gone; `links` every absolute anchor in document order; `extract` is the page read
|
|
183
|
-
* into the fields the caller asked for, which is the one format a model writes
|
|
196
|
+
* into the fields the caller asked for, which is the one format a model writes; `raw` is the
|
|
197
|
+
* response's own bytes, the file a browser would save. */
|
|
184
198
|
export const SCRAPE_FORMATS = [
|
|
185
199
|
"markdown",
|
|
186
200
|
"html",
|
|
@@ -189,6 +203,7 @@ export const SCRAPE_FORMATS = [
|
|
|
189
203
|
"screenshot",
|
|
190
204
|
"pdf",
|
|
191
205
|
"extract",
|
|
206
|
+
"raw",
|
|
192
207
|
] as const;
|
|
193
208
|
|
|
194
209
|
export type ScrapeFormat = (typeof SCRAPE_FORMATS)[number];
|
|
@@ -210,10 +225,13 @@ export type SearchTopic = (typeof SEARCH_TOPICS)[number];
|
|
|
210
225
|
|
|
211
226
|
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
212
227
|
* one it was, because a result set's flavour follows its index. */
|
|
213
|
-
export const SEARCH_ENGINES = ["bing", "brave"] as const;
|
|
228
|
+
export const SEARCH_ENGINES = ["bing", "brave", "duckduckgo"] as const;
|
|
214
229
|
|
|
215
230
|
export type SearchEngine = (typeof SEARCH_ENGINES)[number];
|
|
216
231
|
|
|
232
|
+
/** Where a map's URLs came from. With none found, the sources that were read. */
|
|
233
|
+
export type MapSource = "sitemap" | "links" | "both";
|
|
234
|
+
|
|
217
235
|
/** A batch's life. `queued` is accepted but not yet picked up; `running` has at least one
|
|
218
236
|
* item in flight. The three end states answer the same question the session statuses do —
|
|
219
237
|
* who ended it: `completed` ran every item to a verdict (some of which may be failures,
|
|
@@ -275,6 +293,20 @@ export interface ScrapeMetadataDto {
|
|
|
275
293
|
* it so a caller can retry with `engine: "browser"` instead of trusting an empty page.
|
|
276
294
|
* Always false from the browser engine, which ran the scripts. */
|
|
277
295
|
needsJs: boolean;
|
|
296
|
+
/** How many pages the PDF has, when a PDF was read into text. Absent for anything else,
|
|
297
|
+
* and when only "raw" was asked for. */
|
|
298
|
+
pages?: number;
|
|
299
|
+
/** True when the PDF has more pages than we read: the first 200 are in the answer and the
|
|
300
|
+
* rest are not. Absent otherwise. */
|
|
301
|
+
truncated?: boolean;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** A file, exactly as the site sent it. */
|
|
305
|
+
export interface ScrapeRawDto {
|
|
306
|
+
/** The bytes, base64-encoded. */
|
|
307
|
+
base64: string;
|
|
308
|
+
/** The response's `content-type`, media type only: `application/pdf`. */
|
|
309
|
+
contentType: string | null;
|
|
278
310
|
}
|
|
279
311
|
|
|
280
312
|
/** `POST /v1/scrape`'s answer. Only the requested formats are present; everything else is
|
|
@@ -307,6 +339,9 @@ export interface ScrapeResultDto {
|
|
|
307
339
|
links?: string[];
|
|
308
340
|
screenshot?: string;
|
|
309
341
|
pdf?: string;
|
|
342
|
+
/** The response's own bytes, as a browser would save them (any compression undone), when
|
|
343
|
+
* `raw` was in `formats`. Never cut short: a file over the limit is refused instead. */
|
|
344
|
+
raw?: ScrapeRawDto;
|
|
310
345
|
/** The fields asked for in `extract`, read off the page by a model. Null when the model
|
|
311
346
|
* could not answer — that credit is handed back and the page still stands. Present only
|
|
312
347
|
* when `extract` was in `formats`. */
|
|
@@ -449,6 +484,43 @@ export interface CreatedBatchDto extends BatchDto {
|
|
|
449
484
|
* delivery somebody copied can be sent to you again. */
|
|
450
485
|
webhookSecret?: string;
|
|
451
486
|
}
|
|
487
|
+
// ── from packages/shared/src/dto/map.ts ──────────────────────────────
|
|
488
|
+
|
|
489
|
+
/** One URL a site lists, normalised: no fragment, no `utm_*` parameters, the query sorted,
|
|
490
|
+
* no trailing slash. */
|
|
491
|
+
export interface MapUrlDto {
|
|
492
|
+
url: string;
|
|
493
|
+
/** When the sitemap says the page last changed, as the sitemap wrote it (a W3C date, often
|
|
494
|
+
* just the day). Absent when it did not say, or when the URL came from the page's links. */
|
|
495
|
+
lastmod?: string;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/** `POST /v1/map`'s answer. */
|
|
499
|
+
export interface MapResultDto {
|
|
500
|
+
/** Every URL found, once each, sitemap URLs first in the order the site listed them. */
|
|
501
|
+
urls: MapUrlDto[];
|
|
502
|
+
/** Where they came from. With none found, the sources that were read. */
|
|
503
|
+
source: MapSource;
|
|
504
|
+
/** True when the site may list more than this answer holds: `limit` cut the list, or the
|
|
505
|
+
* robots.txt or a sitemap was not read to its end (the site refused it, its size, the
|
|
506
|
+
* 25-document cap, or the time limit). */
|
|
507
|
+
truncated: boolean;
|
|
508
|
+
pool: ProxyPool;
|
|
509
|
+
/** What the managed pool carried for every document the map read, decompressed. The
|
|
510
|
+
* first megabyte is free; past it, bandwidth is metered like any request's. */
|
|
511
|
+
bytes: number;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** What a map would cost, without making it. */
|
|
515
|
+
export interface MapQuoteDto {
|
|
516
|
+
pool: ProxyPool;
|
|
517
|
+
/** Credits a map charges, whatever it finds. */
|
|
518
|
+
cost: number;
|
|
519
|
+
/** Managed-pool megabytes included before `perMb` applies. 0 on BYO. */
|
|
520
|
+
freeMb: number;
|
|
521
|
+
/** Credits per additional megabyte through the managed pool. 0 on BYO. */
|
|
522
|
+
perMb: number;
|
|
523
|
+
}
|
|
452
524
|
// ── from packages/shared/src/dto/search.ts ───────────────────────────
|
|
453
525
|
|
|
454
526
|
/** One hit, as the index ranked it. The array's ORDER is the ranking — there is no score,
|
|
@@ -10,6 +10,8 @@ import type {
|
|
|
10
10
|
BatchQuoteDto,
|
|
11
11
|
CreatedBatchDto,
|
|
12
12
|
LiveSessionDto,
|
|
13
|
+
MapQuoteDto,
|
|
14
|
+
MapResultDto,
|
|
13
15
|
ScrapeQuoteDto,
|
|
14
16
|
ScrapeResultDto,
|
|
15
17
|
SearchResultDto,
|
|
@@ -26,7 +28,9 @@ import type {
|
|
|
26
28
|
GetSessionParams,
|
|
27
29
|
ListBatchesParams,
|
|
28
30
|
ListSessionsParams,
|
|
31
|
+
MapSiteParams,
|
|
29
32
|
QuoteBatchParams,
|
|
33
|
+
QuoteMapParams,
|
|
30
34
|
QuoteScrapeParams,
|
|
31
35
|
QuoteSessionParams,
|
|
32
36
|
ScrapeParams,
|
|
@@ -34,7 +38,7 @@ import type {
|
|
|
34
38
|
} from "./params.ts";
|
|
35
39
|
|
|
36
40
|
/** The version this client reports in `x-scrapa-client`. */
|
|
37
|
-
export const SDK_VERSION = "0.
|
|
41
|
+
export const SDK_VERSION = "0.4.0";
|
|
38
42
|
|
|
39
43
|
/** The hosted API. */
|
|
40
44
|
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
@@ -50,12 +54,14 @@ export abstract class GeneratedOperations {
|
|
|
50
54
|
* Scrape a page
|
|
51
55
|
*
|
|
52
56
|
* Read one web page and hand it back as markdown, HTML, plain text, its links, a screenshot, a
|
|
53
|
-
* PDF
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* and
|
|
57
|
+
* PDF, or the fields you ask for, read off the page by a model. `engine: "auto"` (the default)
|
|
58
|
+
* makes a plain HTTP request first and re-reads the page in a real Chromium when the body
|
|
59
|
+
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
60
|
+
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
61
|
+
* file exactly as the site sent it, ask for `formats: ["raw"]`. A link to a PDF or a Word file
|
|
62
|
+
* is read into text too, headings and paragraphs included. Every request leaves through a
|
|
63
|
+
* residential proxy, or through your own with `proxy: { pool: "custom", server }`. The answer
|
|
64
|
+
* says which engine replied, where the page finally was and what it cost.
|
|
59
65
|
*
|
|
60
66
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
61
67
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
@@ -84,11 +90,11 @@ export abstract class GeneratedOperations {
|
|
|
84
90
|
* Price a scrape
|
|
85
91
|
*
|
|
86
92
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
87
|
-
* real call meets
|
|
93
|
+
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
88
94
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
89
95
|
* out to need one.
|
|
90
96
|
*
|
|
91
|
-
* Credits: 0 credits
|
|
97
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
92
98
|
*
|
|
93
99
|
* `POST /v1/scrape/quote`
|
|
94
100
|
*/
|
|
@@ -108,15 +114,75 @@ export abstract class GeneratedOperations {
|
|
|
108
114
|
);
|
|
109
115
|
}
|
|
110
116
|
|
|
117
|
+
/**
|
|
118
|
+
* List a site's URLs
|
|
119
|
+
*
|
|
120
|
+
* Hand back every URL a site lists, without reading the pages: what its robots.txt and
|
|
121
|
+
* sitemaps name (gzipped ones too, and indexes of sitemaps), plus the links on the page you
|
|
122
|
+
* named. Each URL comes back once, without tracking parameters or a trailing slash, with
|
|
123
|
+
* `lastmod` when the sitemap gives one. Only the host you named is kept unless you set
|
|
124
|
+
* `includeSubdomains`, and `search` keeps the URLs that contain a word. To read a page, pass
|
|
125
|
+
* its URL to `scrape`.
|
|
126
|
+
*
|
|
127
|
+
* Credits: 2 credits, plus managed-proxy bandwidth past the first megabyte, every file the map
|
|
128
|
+
* read counted together. A site that lists nothing is charged, because it answered. A site
|
|
129
|
+
* that refused every file, and anything we broke, cost nothing.
|
|
130
|
+
*
|
|
131
|
+
* `POST /v1/map`
|
|
132
|
+
*/
|
|
133
|
+
map(
|
|
134
|
+
params: MapSiteParams,
|
|
135
|
+
opts?: Omit<RequestOptions, "idempotencyKey">,
|
|
136
|
+
): Promise<MapResultDto> {
|
|
137
|
+
return this.request(
|
|
138
|
+
{
|
|
139
|
+
method: "POST",
|
|
140
|
+
path: "/map",
|
|
141
|
+
repeatable: false,
|
|
142
|
+
proxy: ["pool", "server", "country"],
|
|
143
|
+
},
|
|
144
|
+
params,
|
|
145
|
+
opts,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Price a map
|
|
151
|
+
*
|
|
152
|
+
* What the same map would cost, without making it. Free, and refused by the same guards a real
|
|
153
|
+
* call meets. Bandwidth is priced per megabyte because nobody knows a site's sitemap size
|
|
154
|
+
* before reading it.
|
|
155
|
+
*
|
|
156
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
157
|
+
*
|
|
158
|
+
* `POST /v1/map/quote`
|
|
159
|
+
*/
|
|
160
|
+
quoteMap(
|
|
161
|
+
params: QuoteMapParams,
|
|
162
|
+
opts?: Omit<RequestOptions, "idempotencyKey">,
|
|
163
|
+
): Promise<MapQuoteDto> {
|
|
164
|
+
return this.request(
|
|
165
|
+
{
|
|
166
|
+
method: "POST",
|
|
167
|
+
path: "/map/quote",
|
|
168
|
+
repeatable: true,
|
|
169
|
+
proxy: ["pool", "server", "country"],
|
|
170
|
+
},
|
|
171
|
+
params,
|
|
172
|
+
opts,
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
111
176
|
/**
|
|
112
177
|
* Search the web
|
|
113
178
|
*
|
|
114
|
-
* Ask the web a question and get back ranked results
|
|
115
|
-
* holds
|
|
116
|
-
* cache. `country` picks both the exit and the market, so `br` returns what
|
|
117
|
-
* would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
118
|
-
* when you want dated journalism, and add `answer: true` when you want a
|
|
119
|
-
* over the results instead of writing it yourself. To read a result, pass
|
|
179
|
+
* Ask the web a question and get back ranked results: title, URL and the snippet the index
|
|
180
|
+
* holds. They are read live off a real search front through a residential exit, never a
|
|
181
|
+
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
182
|
+
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
183
|
+
* for `topic: "news"` when you want dated journalism, and add `answer: true` when you want a
|
|
184
|
+
* paragraph written over the results instead of writing it yourself. To read a result, pass
|
|
185
|
+
* its URL to `scrape`.
|
|
120
186
|
*
|
|
121
187
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
122
188
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -146,8 +212,8 @@ export abstract class GeneratedOperations {
|
|
|
146
212
|
* Start a batch
|
|
147
213
|
*
|
|
148
214
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
149
|
-
* options you pass here
|
|
150
|
-
* `scrape
|
|
215
|
+
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
216
|
+
* single `scrape`, and a page the site blocked costs nothing here too. Nothing is charged
|
|
151
217
|
* when you submit: a batch spends as it goes, so use `quote` for the ceiling. Poll `get`
|
|
152
218
|
* for progress, or pass a `webhook` and we will call you instead. Results are readable for
|
|
153
219
|
* 24 hours.
|
|
@@ -179,9 +245,9 @@ export abstract class GeneratedOperations {
|
|
|
179
245
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
180
246
|
* URL at the ceiling one page could reach, so the real bill comes in lower by whatever the
|
|
181
247
|
* web refuses for free and by every page that never needed a browser. Bandwidth is on top
|
|
182
|
-
* and cannot be quoted
|
|
248
|
+
* and cannot be quoted, because it depends on what the pages weigh.
|
|
183
249
|
*
|
|
184
|
-
* Credits: 0 credits
|
|
250
|
+
* Credits: 0 credits. Pricing a batch is free.
|
|
185
251
|
*
|
|
186
252
|
* `POST /v1/batches/quote`
|
|
187
253
|
*/
|
|
@@ -202,10 +268,10 @@ export abstract class GeneratedOperations {
|
|
|
202
268
|
/**
|
|
203
269
|
* List your batches
|
|
204
270
|
*
|
|
205
|
-
* Your most recent batches, newest first
|
|
206
|
-
* what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
271
|
+
* Your most recent batches, newest first, with what each one is doing, how far it has got
|
|
272
|
+
* and what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
207
273
|
*
|
|
208
|
-
* Credits: 0 credits
|
|
274
|
+
* Credits: 0 credits. Listing batches is free.
|
|
209
275
|
*
|
|
210
276
|
* `GET /v1/batches`
|
|
211
277
|
*/
|
|
@@ -218,13 +284,13 @@ export abstract class GeneratedOperations {
|
|
|
218
284
|
*
|
|
219
285
|
* Where a batch got to: whether it is still running, how many of its URLs now have a
|
|
220
286
|
* verdict, how many of those the web or our policy refused, how many were never attempted,
|
|
221
|
-
* and what it has spent so far. Ask for `items` to get the results themselves
|
|
287
|
+
* and what it has spent so far. Ask for `items` to get the results themselves: the pages
|
|
222
288
|
* and the refusals, in the order you submitted them, each one the same answer a single
|
|
223
|
-
* `scrape` would have given
|
|
224
|
-
* whose results have expired still answers with everything else, because somebody
|
|
225
|
-
* for those pages must never be told the batch does not exist.
|
|
289
|
+
* `scrape` would have given. You get up to 200 at a time, from wherever `from` says. A
|
|
290
|
+
* batch whose results have expired still answers with everything else, because somebody
|
|
291
|
+
* who paid for those pages must never be told the batch does not exist.
|
|
226
292
|
*
|
|
227
|
-
* Credits: 0 credits
|
|
293
|
+
* Credits: 0 credits. Reading a batch is free.
|
|
228
294
|
*
|
|
229
295
|
* `GET /v1/batches/{id}`
|
|
230
296
|
*/
|
|
@@ -235,12 +301,12 @@ export abstract class GeneratedOperations {
|
|
|
235
301
|
/**
|
|
236
302
|
* Cancel a batch
|
|
237
303
|
*
|
|
238
|
-
* Stop a batch. A URL already being read finishes and is charged
|
|
239
|
-
* already paying for
|
|
240
|
-
*
|
|
241
|
-
*
|
|
304
|
+
* Stop a batch. A URL already being read finishes and is charged, because it is a page we
|
|
305
|
+
* are already paying for. Every URL not yet started becomes `skipped`, which costs nothing
|
|
306
|
+
* and is not a verdict about the URL: submit those positions again whenever you like. Safe
|
|
307
|
+
* to repeat; a batch that has already ended answers with how it ended.
|
|
242
308
|
*
|
|
243
|
-
* Credits: 0 credits
|
|
309
|
+
* Credits: 0 credits. Cancelling is free, and it stops the spending.
|
|
244
310
|
*
|
|
245
311
|
* `DELETE /v1/batches/{id}`
|
|
246
312
|
*/
|
|
@@ -264,10 +330,10 @@ export abstract class GeneratedOperations {
|
|
|
264
330
|
* `puppeteer.connect({ browserWSEndpoint: connectUrl })` in Puppeteer. The browser is
|
|
265
331
|
* yours alone for the whole window, every byte it pulls leaves through the exit you
|
|
266
332
|
* picked, and the socket survives our deploys. The URL carries the session's token and is
|
|
267
|
-
* shown once
|
|
333
|
+
* shown once. It cannot be rebuilt from the id.
|
|
268
334
|
*
|
|
269
335
|
* Credits: 2 credits per minute, reserved for the whole window when the session starts and
|
|
270
|
-
* settled by the minutes actually used when it closes
|
|
336
|
+
* settled by the minutes actually used when it closes, so closing early gives the rest
|
|
271
337
|
* back, and a browser that never came up costs nothing at all. Managed-proxy bandwidth
|
|
272
338
|
* past the first megabyte is added at close; your own proxy is never metered.
|
|
273
339
|
*
|
|
@@ -291,10 +357,10 @@ export abstract class GeneratedOperations {
|
|
|
291
357
|
* Price a session
|
|
292
358
|
*
|
|
293
359
|
* What holding a browser for that long would cost, without holding one. `maxCost` is the
|
|
294
|
-
* whole window at the per-minute rate
|
|
295
|
-
* not use come back when the session closes.
|
|
360
|
+
* whole window at the per-minute rate. It is a ceiling, not a bill, because the minutes
|
|
361
|
+
* you do not use come back when the session closes.
|
|
296
362
|
*
|
|
297
|
-
* Credits: 0 credits
|
|
363
|
+
* Credits: 0 credits. Pricing a session is free.
|
|
298
364
|
*
|
|
299
365
|
* `POST /v1/sessions/quote`
|
|
300
366
|
*/
|
|
@@ -315,10 +381,10 @@ export abstract class GeneratedOperations {
|
|
|
315
381
|
/**
|
|
316
382
|
* List your sessions
|
|
317
383
|
*
|
|
318
|
-
* Your most recent sessions, newest first
|
|
384
|
+
* Your most recent sessions, newest first: the live ones and the ones that have ended,
|
|
319
385
|
* with what each of them cost.
|
|
320
386
|
*
|
|
321
|
-
* Credits: 0 credits
|
|
387
|
+
* Credits: 0 credits. Listing sessions is free.
|
|
322
388
|
*
|
|
323
389
|
* `GET /v1/sessions`
|
|
324
390
|
*/
|
|
@@ -331,10 +397,10 @@ export abstract class GeneratedOperations {
|
|
|
331
397
|
* Read one session
|
|
332
398
|
*
|
|
333
399
|
* Where a session got to: whether it is still live, when it expires, what it has pulled
|
|
334
|
-
* and
|
|
400
|
+
* and, once it has ended, what it finally cost. The connect URL is not repeated here; it
|
|
335
401
|
* is shown once, when the session starts.
|
|
336
402
|
*
|
|
337
|
-
* Credits: 0 credits
|
|
403
|
+
* Credits: 0 credits. Reading a session is free.
|
|
338
404
|
*
|
|
339
405
|
* `GET /v1/sessions/{id}`
|
|
340
406
|
*/
|
|
@@ -347,8 +413,8 @@ export abstract class GeneratedOperations {
|
|
|
347
413
|
* Close a session
|
|
348
414
|
*
|
|
349
415
|
* Stop the browser and settle the bill: the minutes you held it are kept and the rest of
|
|
350
|
-
* the reservation goes back. Safe to repeat
|
|
351
|
-
*
|
|
416
|
+
* the reservation goes back. Safe to repeat: a session that has already ended answers with
|
|
417
|
+
* how it ended.
|
|
352
418
|
*
|
|
353
419
|
* Credits: Closing is free. It is where the session's own bill lands: the unused minutes
|
|
354
420
|
* are refunded and the managed pool's bandwidth is charged.
|