@detiq/crawler 1.1.13 → 1.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/Dockerfile +1 -1
  2. package/dist/accessibility-tree-crawler.js +156 -0
  3. package/dist/accessibility-tree-crawler.js.map +7 -0
  4. package/dist/advanced-ui-probes.js +310 -0
  5. package/dist/advanced-ui-probes.js.map +7 -0
  6. package/dist/content-chunker.js +2 -2
  7. package/dist/content-chunker.js.map +2 -2
  8. package/dist/crawl-session-manager.js +145 -0
  9. package/dist/crawl-session-manager.js.map +7 -0
  10. package/dist/crawler.js +1142 -263
  11. package/dist/crawler.js.map +2 -2
  12. package/dist/crud-interaction.js +264 -0
  13. package/dist/crud-interaction.js.map +7 -0
  14. package/dist/dropdown-enumerator.js +230 -0
  15. package/dist/dropdown-enumerator.js.map +7 -0
  16. package/dist/filter-pagination-crawler.js +299 -0
  17. package/dist/filter-pagination-crawler.js.map +7 -0
  18. package/dist/frontier.js +5 -3
  19. package/dist/frontier.js.map +2 -2
  20. package/dist/gap-pass.js +47 -14
  21. package/dist/gap-pass.js.map +2 -2
  22. package/dist/http-engine.js +28 -15
  23. package/dist/http-engine.js.map +2 -2
  24. package/dist/index.js +15 -1
  25. package/dist/index.js.map +2 -2
  26. package/dist/job-queue.js +12 -5
  27. package/dist/job-queue.js.map +2 -2
  28. package/dist/locator-extractor.js +23 -0
  29. package/dist/locator-extractor.js.map +2 -2
  30. package/dist/multi-context-handler.js +182 -0
  31. package/dist/multi-context-handler.js.map +7 -0
  32. package/dist/nested-window-crawler.js +172 -0
  33. package/dist/nested-window-crawler.js.map +7 -0
  34. package/dist/network-action-correlator.js +209 -0
  35. package/dist/network-action-correlator.js.map +7 -0
  36. package/dist/performance-metrics-collector.js +155 -0
  37. package/dist/performance-metrics-collector.js.map +7 -0
  38. package/dist/performance-metrics.js.map +2 -2
  39. package/dist/policies/crawl-safety.js +5 -1
  40. package/dist/policies/crawl-safety.js.map +2 -2
  41. package/dist/profiles/domain-profile.js +21 -1
  42. package/dist/profiles/domain-profile.js.map +2 -2
  43. package/dist/rate-limiter.js +2 -2
  44. package/dist/rate-limiter.js.map +2 -2
  45. package/dist/screen-reader.js +1 -0
  46. package/dist/screen-reader.js.map +2 -2
  47. package/dist/service-worker-interceptor.js +136 -0
  48. package/dist/service-worker-interceptor.js.map +7 -0
  49. package/dist/session-capture.js +4 -2
  50. package/dist/session-capture.js.map +2 -2
  51. package/dist/shadow-dom-crawler.js +132 -0
  52. package/dist/shadow-dom-crawler.js.map +7 -0
  53. package/dist/sitemap.js +16 -10
  54. package/dist/sitemap.js.map +2 -2
  55. package/dist/state-graph.js +193 -0
  56. package/dist/state-graph.js.map +7 -0
  57. package/dist/stealth/human-behavior.js +36 -3
  58. package/dist/stealth/human-behavior.js.map +2 -2
  59. package/dist/url-pattern-completer.js +48 -14
  60. package/dist/url-pattern-completer.js.map +2 -2
  61. package/dist/visual-diff-verdict.js +3 -1
  62. package/dist/visual-diff-verdict.js.map +2 -2
  63. package/package.json +2 -1
  64. package/src/accessibility-tree-crawler.ts +246 -0
  65. package/src/advanced-ui-probes.ts +470 -0
  66. package/src/crawl-session-manager.ts +231 -0
  67. package/src/crawler.ts +1071 -142
  68. package/src/crud-interaction.ts +386 -0
  69. package/src/dropdown-enumerator.ts +312 -0
  70. package/src/filter-pagination-crawler.ts +500 -0
  71. package/src/gap-pass.ts +58 -16
  72. package/src/http-engine.ts +29 -15
  73. package/src/index.ts +14 -1
  74. package/src/job-queue.ts +3 -1
  75. package/src/locator-extractor.ts +45 -0
  76. package/src/multi-context-handler.ts +296 -0
  77. package/src/nested-window-crawler.ts +310 -0
  78. package/src/network-action-correlator.ts +320 -0
  79. package/src/performance-metrics-collector.ts +243 -0
  80. package/src/performance-metrics.ts +1 -1
  81. package/src/screen-reader.ts +4 -1
  82. package/src/service-worker-interceptor.ts +185 -0
  83. package/src/session-capture.ts +3 -0
  84. package/src/shadow-dom-crawler.ts +198 -0
  85. package/src/sitemap.ts +17 -10
  86. package/src/state-graph.ts +280 -0
  87. package/src/stealth/human-behavior.ts +38 -4
  88. package/src/url-pattern-completer.ts +61 -15
  89. package/tsconfig.json +1 -1
  90. package/types/accessibility-tree-crawler.d.ts +44 -0
  91. package/types/advanced-ui-probes.d.ts +47 -0
  92. package/types/crawl-session-manager.d.ts +44 -0
  93. package/types/crawler.d.ts +3 -1
  94. package/types/crud-interaction.d.ts +31 -0
  95. package/types/dropdown-enumerator.d.ts +20 -0
  96. package/types/filter-pagination-crawler.d.ts +52 -0
  97. package/types/gap-pass.d.ts +1 -0
  98. package/types/index.d.ts +14 -1
  99. package/types/job-queue.d.ts +18 -9
  100. package/types/locator-extractor.d.ts +7 -0
  101. package/types/multi-context-handler.d.ts +55 -0
  102. package/types/nested-window-crawler.d.ts +41 -0
  103. package/types/network-action-correlator.d.ts +47 -0
  104. package/types/performance-metrics-collector.d.ts +50 -0
  105. package/types/performance-metrics.d.ts +1 -2
  106. package/types/screen-reader.d.ts +1 -2
  107. package/types/service-worker-interceptor.d.ts +36 -0
  108. package/types/session-capture.d.ts +1 -1
  109. package/types/shadow-dom-crawler.d.ts +35 -0
  110. package/types/sitemap.d.ts +2 -2
  111. package/types/state-graph.d.ts +59 -0
  112. package/types/stealth/human-behavior.d.ts +1 -1
package/types/stealth/human-behavior.d.ts CHANGED
@@ -8,7 +8,7 @@ export declare function infiniteScrollToBottom(page: Page, opts?: {
8
8
  maxScrolls?: number;
9
9
  scrollDelay?: number;
10
10
  stabilizeMs?: number;
11
- }): Promise<{
11
+ }, rawPage?: any): Promise<{
12
12
  scrollCount: number;
13
13
  heightReached: number;
14
14
  }>;