@nitpicker/crawler 0.6.3 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/lib/crawler/fetch-destination.js +11 -5
  2. package/package.json +2 -2
  3. package/lib/archive/common-queries.d.ts +0 -14
  4. package/lib/archive/common-queries.js +0 -40
  5. package/lib/archive/filesystem/index.d.ts +0 -17
  6. package/lib/archive/filesystem/index.js +0 -17
  7. package/lib/archive/filesystem/utils.d.ts +0 -109
  8. package/lib/archive/filesystem/utils.js +0 -185
  9. package/lib/archive/filesystem/zip.d.ts +0 -29
  10. package/lib/archive/filesystem/zip.js +0 -53
  11. package/lib/archive/index.d.ts +0 -6
  12. package/lib/archive/index.js +0 -11
  13. package/lib/crawler/index.d.ts +0 -2
  14. package/lib/crawler/index.js +0 -2
  15. package/lib/crawler/network.d.ts +0 -30
  16. package/lib/crawler/network.js +0 -226
  17. package/lib/crawler/result-handler.d.ts +0 -118
  18. package/lib/crawler/result-handler.js +0 -153
  19. package/lib/crawler/speculative-pagination.d.ts +0 -52
  20. package/lib/crawler/speculative-pagination.js +0 -215
  21. package/lib/crawler/url-filter.d.ts +0 -56
  22. package/lib/crawler/url-filter.js +0 -110
  23. package/lib/index.d.ts +0 -16
  24. package/lib/index.js +0 -18
  25. package/lib/qzilla.d.ts +0 -136
  26. package/lib/qzilla.js +0 -292
  27. package/lib/utils/array/index.d.ts +0 -1
  28. package/lib/utils/array/index.js +0 -1
  29. package/lib/utils/async/index.d.ts +0 -1
  30. package/lib/utils/async/index.js +0 -1
  31. package/lib/utils/error/index.d.ts +0 -3
  32. package/lib/utils/error/index.js +0 -2
  33. package/lib/utils/event-emitter/index.d.ts +0 -6
  34. package/lib/utils/event-emitter/index.js +0 -6
  35. package/lib/utils/index.d.ts +0 -5
  36. package/lib/utils/index.js +0 -5
  37. package/lib/utils/network/index.d.ts +0 -1
  38. package/lib/utils/network/index.js +0 -1
  39. package/lib/utils/object/index.d.ts +0 -1
  40. package/lib/utils/object/index.js +0 -1
  41. package/lib/utils/path/index.d.ts +0 -1
  42. package/lib/utils/path/index.js +0 -1
  43. package/lib/utils/path/safe-filepath.d.ts +0 -7
  44. package/lib/utils/path/safe-filepath.js +0 -12
  45. package/lib/utils/regexp/index.d.ts +0 -1
  46. package/lib/utils/regexp/index.js +0 -1
  47. package/lib/utils/retryable/index.d.ts +0 -2
  48. package/lib/utils/retryable/index.js +0 -1
  49. package/lib/utils/sort/index.d.ts +0 -14
  50. package/lib/utils/sort/index.js +0 -61
  51. package/lib/utils/sort/remove-matches.d.ts +0 -9
  52. package/lib/utils/sort/remove-matches.js +0 -23
  53. package/lib/utils/types/index.d.ts +0 -1
  54. package/lib/utils/types/index.js +0 -1
  55. package/lib/utils/url/index.d.ts +0 -5
  56. package/lib/utils/url/index.js +0 -5
  57. package/lib/utils/url/is-lower-layer.d.ts +0 -15
  58. package/lib/utils/url/is-lower-layer.js +0 -55
  59. package/lib/utils/url/parse-url.d.ts +0 -11
  60. package/lib/utils/url/parse-url.js +0 -20
  61. package/lib/utils/url/path-match.d.ts +0 -11
  62. package/lib/utils/url/path-match.js +0 -18
  63. package/lib/utils/url/sort-url.d.ts +0 -10
  64. package/lib/utils/url/sort-url.js +0 -24
  65. package/lib/utils/url/url-partial-match.d.ts +0 -11
  66. package/lib/utils/url/url-partial-match.js +0 -32
@@ -1,18 +0,0 @@
1
- import micromatch from 'micromatch';
2
- import { parseUrl } from './parse-url.js';
3
- /**
4
- * Tests whether the pathname portion of a URL matches a given glob pattern.
5
- * Uses micromatch for glob matching.
6
- * @param targetPath - The URL string or ExURL whose pathname will be tested.
7
- * @param pattern - The glob pattern to match against the pathname (e.g., `"/docs/**"`).
8
- * @param options - Optional URL parsing options.
9
- * @returns `true` if the URL's pathname matches the pattern; `false` otherwise
10
- * (also returns `false` if the URL cannot be parsed).
11
- */
12
- export function pathMatch(targetPath, pattern, options) {
13
- const url = typeof targetPath === 'string' ? parseUrl(targetPath, options) : targetPath;
14
- if (!url) {
15
- return false;
16
- }
17
- return micromatch.isMatch(url.pathname || '', pattern || '/');
18
- }
@@ -1,10 +0,0 @@
1
- import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
2
- /**
3
- * Parses, deduplicates, and sorts a list of URL strings using natural URL sorting.
4
- * Duplicate URLs (by normalized href) are removed before sorting.
5
- * @param list - An array of URL strings to sort.
6
- * @param options - Optional URL parsing options.
7
- * @returns A sorted array of ExURL objects with duplicates removed, ordered by
8
- * natural URL sort (hostname, path hierarchy, basename, extension, query, hash).
9
- */
10
- export declare function sortUrl(list: string[], options?: ParseURLOptions): ExURL[];
@@ -1,24 +0,0 @@
1
- import { pathComparator } from '@d-zero/shared/sort/path';
2
- import { parseUrl } from './parse-url.js';
3
- /**
4
- * Parses, deduplicates, and sorts a list of URL strings using natural URL sorting.
5
- * Duplicate URLs (by normalized href) are removed before sorting.
6
- * @param list - An array of URL strings to sort.
7
- * @param options - Optional URL parsing options.
8
- * @returns A sorted array of ExURL objects with duplicates removed, ordered by
9
- * natural URL sort (hostname, path hierarchy, basename, extension, query, hash).
10
- */
11
- export function sortUrl(list, options) {
12
- const map = new Map();
13
- for (const url of list) {
14
- if (map.has(url)) {
15
- continue;
16
- }
17
- const parsedUrl = parseUrl(url, options);
18
- if (!parsedUrl) {
19
- continue;
20
- }
21
- map.set(parsedUrl.href, parsedUrl);
22
- }
23
- return [...map.values()].toSorted((a, b) => pathComparator(a.href, b.href));
24
- }
@@ -1,11 +0,0 @@
1
- import type { ExURL } from '@d-zero/shared/parse-url';
2
- /**
3
- * Checks whether a URL partially matches a pattern URL. The match succeeds if both the
4
- * hostname (case-insensitive) and the pathname match. The pathname matches if it is exactly
5
- * equal to the pattern's pathname or if it falls under the pattern's pathname as a sub-path.
6
- * @param url - The URL string or ExURL to test.
7
- * @param pattern - The pattern URL string to match against (must include hostname and path).
8
- * @returns `true` if the URL's hostname matches the pattern's hostname and the URL's pathname
9
- * is equal to or nested under the pattern's pathname; `false` otherwise.
10
- */
11
- export declare function urlPartialMatch(url: string | ExURL, pattern: string): boolean;
@@ -1,32 +0,0 @@
1
- import { parseUrl } from './parse-url.js';
2
- import { pathMatch } from './path-match.js';
3
- /**
4
- * Checks whether a URL partially matches a pattern URL. The match succeeds if both the
5
- * hostname (case-insensitive) and the pathname match. The pathname matches if it is exactly
6
- * equal to the pattern's pathname or if it falls under the pattern's pathname as a sub-path.
7
- * @param url - The URL string or ExURL to test.
8
- * @param pattern - The pattern URL string to match against (must include hostname and path).
9
- * @returns `true` if the URL's hostname matches the pattern's hostname and the URL's pathname
10
- * is equal to or nested under the pattern's pathname; `false` otherwise.
11
- */
12
- export function urlPartialMatch(url, pattern) {
13
- const target = parseUrl(url);
14
- if (!target) {
15
- return false;
16
- }
17
- const patternUrl = parseUrl(pattern);
18
- if (!patternUrl) {
19
- return false;
20
- }
21
- const { hostname, pathname } = patternUrl;
22
- if (hostname.toLowerCase() !== target.hostname.toLowerCase()) {
23
- return false;
24
- }
25
- if (pathname === target.pathname) {
26
- return true;
27
- }
28
- if (pathMatch(url, `${pathname}/**/*`)) {
29
- return true;
30
- }
31
- return false;
32
- }