@happyvertical/utils 0.80.0 → 0.80.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,394 +1,452 @@
1
+ import { A as ApiError, B as enableLogging, C as prettyDate, D as urlFilename, E as snakeCase, F as NetworkError, G as extractAllCodeBlocks, H as setLogger, I as ParsingError, J as extractJSON, K as extractCodeBlock, L as TimeoutError, M as DatabaseError, N as ErrorCode, O as urlPath, P as FileError, Q as toScreamingSnakeCase, R as ValidationError, S as pluralizeWord, T as sleep, U as isSafeCode, V as getLogger, W as validateCode, X as loadEnvConfig, Y as convertType, Z as toCamelCase, _ as logTicker, a as domainToCamel, b as parseAmazonDateString, c as isArray, d as isPlural, f as isSingular, g as keysToSnake, h as keysToCamel, i as dateInString, j as BaseError, k as waitFor, l as isCuid, m as isValidDate, n as camelCase, o as formatDate, p as isUrl, q as extractFunctionDefinition, r as createId, s as getTempDirectory, t as addInterval, u as isPlainObject, v as makeId, w as singularize, x as parseDate, y as makeSlug, z as disableLogging } from "./chunks/universal-DOIuUel_.js";
1
2
  import { basename } from "node:path";
2
3
  import { parseArgs } from "node:util";
3
- import { A, B, D, E, F, N, P, T, V, a, c, b, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, C, G, H, I, J, K, L, M, O, Q, R, S, U, W, X, Y } from "./chunks/universal-BeseWxvS.js";
4
4
  import * as vm from "node:vm";
5
5
  import { createHash } from "node:crypto";
6
- import { isCuid } from "@paralleldrive/cuid2";
6
+ //#region src/cli/parse-args.ts
7
+ /**
8
+ * CLI Argument Parsing Utility
9
+ *
10
+ * Provides robust parsing for multi-word commands, options, and arguments.
11
+ * Supports Node.js util.parseArgs for option parsing with fallback.
12
+ */
13
+ /**
14
+ * Parse command line arguments with support for multi-word commands
15
+ *
16
+ * Handles:
17
+ * - Multi-word commands (up to 3 words: "gnode create", "foo bar baz")
18
+ * - Command aliases
19
+ * - Positional arguments
20
+ * - Options using Node.js util.parseArgs
21
+ * - Global flags (--help, --version)
22
+ * - Automatic removal of node/script paths
23
+ *
24
+ * @param argv - Process argv or custom argument array
25
+ * @param commands - Array of command definitions to match against
26
+ * @param builtInCommands - Optional map of additional built-in commands
27
+ * @returns Parsed arguments with command, args, and options
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const parsed = parseCliArgs(
32
+ * ['node', 'cli.js', 'gnode', 'create', 'my-site', '--template=town'],
33
+ * commands
34
+ * );
35
+ * // { command: 'gnode create', args: ['my-site'], options: { template: 'town' } }
36
+ * ```
37
+ */
7
38
  function parseCliArgs(argv, commands, builtInCommands = {}) {
8
- let args = argv;
9
- if (args.length > 0 && basename(args[0]) === "node") {
10
- args = args.slice(1);
11
- }
12
- if (args.length > 0 && args[0].endsWith(".js")) {
13
- args = args.slice(1);
14
- }
15
- if (args.length === 0) {
16
- return { args: [], options: {} };
17
- }
18
- if (args.includes("--help")) {
19
- return { command: "help", args: [], options: {} };
20
- }
21
- if (args.includes("--version")) {
22
- return { command: "version", args: [], options: {} };
23
- }
24
- let matchedCommand;
25
- let commandName;
26
- let commandWordCount = 0;
27
- for (let i2 = Math.min(3, args.length); i2 > 0; i2--) {
28
- const possibleCommand = args.slice(0, i2).join(" ");
29
- const found = builtInCommands[possibleCommand] || commands.find(
30
- (cmd) => cmd.name === possibleCommand || cmd.aliases?.includes(possibleCommand)
31
- );
32
- if (found) {
33
- matchedCommand = found;
34
- commandName = possibleCommand;
35
- commandWordCount = i2;
36
- break;
37
- }
38
- }
39
- if (!commandName && args.length > 0) {
40
- commandName = args[0];
41
- commandWordCount = 1;
42
- matchedCommand = commands.find(
43
- (cmd) => cmd.name === commandName || cmd.aliases?.includes(commandName)
44
- );
45
- }
46
- if (!matchedCommand) {
47
- if (args.includes("-h")) {
48
- return { command: "help", args: [], options: {} };
49
- }
50
- if (args.includes("-v")) {
51
- return { command: "version", args: [], options: {} };
52
- }
53
- return {
54
- command: commandName,
55
- args: args.slice(1).filter((arg) => !arg.startsWith("-")),
56
- options: {}
57
- };
58
- }
59
- const parseConfig = {
60
- args: args.slice(commandWordCount),
61
- options: {},
62
- strict: false,
63
- // Allow unknown options
64
- allowPositionals: true
65
- // Required for mixing positional args and options
66
- };
67
- const numberOptions = /* @__PURE__ */ new Set();
68
- if (matchedCommand.options) {
69
- for (const [name, option] of Object.entries(matchedCommand.options)) {
70
- const nodeType = option.type === "boolean" ? "boolean" : "string";
71
- if (option.type === "number") {
72
- numberOptions.add(name);
73
- }
74
- parseConfig.options[name] = {
75
- type: nodeType
76
- };
77
- if (option.default !== void 0) {
78
- parseConfig.options[name].default = option.type === "number" ? String(option.default) : option.default;
79
- }
80
- if (option.short) {
81
- parseConfig.options[name].short = option.short;
82
- }
83
- }
84
- }
85
- try {
86
- const parsed = parseArgs(parseConfig);
87
- const options = parsed.values || {};
88
- for (const name of numberOptions) {
89
- if (name in options && options[name] !== void 0) {
90
- const rawValue = options[name];
91
- if (rawValue === "") {
92
- throw new Error(`Invalid number for --${name}: "${rawValue}"`);
93
- }
94
- const value = Number(rawValue);
95
- if (Number.isNaN(value)) {
96
- throw new Error(`Invalid number for --${name}: "${rawValue}"`);
97
- }
98
- options[name] = value;
99
- }
100
- }
101
- return {
102
- command: commandName,
103
- args: parsed.positionals || [],
104
- options
105
- };
106
- } catch (error) {
107
- if (error instanceof Error && error.message.startsWith("Invalid number")) {
108
- throw error;
109
- }
110
- return {
111
- command: commandName,
112
- args: args.slice(commandWordCount).filter((arg) => !arg.startsWith("-")),
113
- options: {}
114
- };
115
- }
39
+ let args = argv;
40
+ if (args.length > 0 && basename(args[0]) === "node") args = args.slice(1);
41
+ if (args.length > 0 && args[0].endsWith(".js")) args = args.slice(1);
42
+ if (args.length === 0) return {
43
+ args: [],
44
+ options: {}
45
+ };
46
+ if (args.includes("--help")) return {
47
+ command: "help",
48
+ args: [],
49
+ options: {}
50
+ };
51
+ if (args.includes("--version")) return {
52
+ command: "version",
53
+ args: [],
54
+ options: {}
55
+ };
56
+ let matchedCommand;
57
+ let commandName;
58
+ let commandWordCount = 0;
59
+ for (let i = Math.min(3, args.length); i > 0; i--) {
60
+ const possibleCommand = args.slice(0, i).join(" ");
61
+ const found = builtInCommands[possibleCommand] || commands.find((cmd) => cmd.name === possibleCommand || cmd.aliases?.includes(possibleCommand));
62
+ if (found) {
63
+ matchedCommand = found;
64
+ commandName = possibleCommand;
65
+ commandWordCount = i;
66
+ break;
67
+ }
68
+ }
69
+ if (!commandName && args.length > 0) {
70
+ commandName = args[0];
71
+ commandWordCount = 1;
72
+ matchedCommand = commands.find((cmd) => cmd.name === commandName || cmd.aliases?.includes(commandName));
73
+ }
74
+ if (!matchedCommand) {
75
+ if (args.includes("-h")) return {
76
+ command: "help",
77
+ args: [],
78
+ options: {}
79
+ };
80
+ if (args.includes("-v")) return {
81
+ command: "version",
82
+ args: [],
83
+ options: {}
84
+ };
85
+ return {
86
+ command: commandName,
87
+ args: args.slice(1).filter((arg) => !arg.startsWith("-")),
88
+ options: {}
89
+ };
90
+ }
91
+ const parseConfig = {
92
+ args: args.slice(commandWordCount),
93
+ options: {},
94
+ strict: false,
95
+ allowPositionals: true
96
+ };
97
+ const numberOptions = /* @__PURE__ */ new Set();
98
+ if (matchedCommand.options) for (const [name, option] of Object.entries(matchedCommand.options)) {
99
+ const nodeType = option.type === "boolean" ? "boolean" : "string";
100
+ if (option.type === "number") numberOptions.add(name);
101
+ parseConfig.options[name] = { type: nodeType };
102
+ if (option.default !== void 0) parseConfig.options[name].default = option.type === "number" ? String(option.default) : option.default;
103
+ if (option.short) parseConfig.options[name].short = option.short;
104
+ }
105
+ try {
106
+ const parsed = parseArgs(parseConfig);
107
+ const options = parsed.values || {};
108
+ for (const name of numberOptions) if (name in options && options[name] !== void 0) {
109
+ const rawValue = options[name];
110
+ if (rawValue === "") throw new Error(`Invalid number for --${name}: "${rawValue}"`);
111
+ const value = Number(rawValue);
112
+ if (Number.isNaN(value)) throw new Error(`Invalid number for --${name}: "${rawValue}"`);
113
+ options[name] = value;
114
+ }
115
+ return {
116
+ command: commandName,
117
+ args: parsed.positionals || [],
118
+ options
119
+ };
120
+ } catch (error) {
121
+ if (error instanceof Error && error.message.startsWith("Invalid number")) throw error;
122
+ return {
123
+ command: commandName,
124
+ args: args.slice(commandWordCount).filter((arg) => !arg.startsWith("-")),
125
+ options: {}
126
+ };
127
+ }
116
128
  }
129
+ //#endregion
130
+ //#region src/cli/parse-flags.ts
117
131
  function parseFlagArgs(args) {
118
- const flags = {};
119
- const positionals = [];
120
- for (let index = 0; index < args.length; index += 1) {
121
- const arg = args[index];
122
- if (arg === "--") {
123
- positionals.push(...args.slice(index + 1));
124
- break;
125
- }
126
- if (!arg.startsWith("--")) {
127
- positionals.push(arg);
128
- continue;
129
- }
130
- const [rawKey, inlineValue] = arg.slice(2).split(/=(.*)/su, 2);
131
- if (rawKey === "") {
132
- throw new Error(
133
- `Invalid flag token "${arg}": flag name is empty. Use --key=value or --key value.`
134
- );
135
- }
136
- const key = rawKey.replace(
137
- /-([a-z])/gu,
138
- (_match, letter) => letter.toUpperCase()
139
- );
140
- if (inlineValue !== void 0) {
141
- flags[key] = inlineValue;
142
- continue;
143
- }
144
- const next = args[index + 1];
145
- if (next !== void 0 && !next.startsWith("--")) {
146
- flags[key] = next;
147
- index += 1;
148
- } else {
149
- flags[key] = true;
150
- }
151
- }
152
- return { flags, positionals };
132
+ const flags = {};
133
+ const positionals = [];
134
+ for (let index = 0; index < args.length; index += 1) {
135
+ const arg = args[index];
136
+ if (arg === "--") {
137
+ positionals.push(...args.slice(index + 1));
138
+ break;
139
+ }
140
+ if (!arg.startsWith("--")) {
141
+ positionals.push(arg);
142
+ continue;
143
+ }
144
+ const [rawKey, inlineValue] = arg.slice(2).split(/=(.*)/su, 2);
145
+ if (rawKey === "") throw new Error(`Invalid flag token "${arg}": flag name is empty. Use --key=value or --key value.`);
146
+ const key = rawKey.replace(/-([a-z])/gu, (_match, letter) => letter.toUpperCase());
147
+ if (inlineValue !== void 0) {
148
+ flags[key] = inlineValue;
149
+ continue;
150
+ }
151
+ const next = args[index + 1];
152
+ if (next !== void 0 && !next.startsWith("--")) {
153
+ flags[key] = next;
154
+ index += 1;
155
+ } else flags[key] = true;
156
+ }
157
+ return {
158
+ flags,
159
+ positionals
160
+ };
153
161
  }
154
- const DEFAULT_BUILTINS = [
155
- "Array",
156
- "Object",
157
- "JSON",
158
- "Math",
159
- "Date",
160
- "String",
161
- "Number",
162
- "Boolean",
163
- "RegExp",
164
- "Set",
165
- "Map",
166
- "WeakSet",
167
- "WeakMap",
168
- "Symbol",
169
- "Promise"
162
+ //#endregion
163
+ //#region src/shared/code/sandbox.ts
164
+ /**
165
+ * Sandbox creation and safe code execution utilities
166
+ *
167
+ * Provides secure execution of generated code in isolated VM contexts with
168
+ * controlled globals, timeouts, and resource constraints.
169
+ */
170
+ /**
171
+ * Default safe built-in objects available in the sandbox
172
+ */
173
+ var DEFAULT_BUILTINS = [
174
+ "Array",
175
+ "Object",
176
+ "JSON",
177
+ "Math",
178
+ "Date",
179
+ "String",
180
+ "Number",
181
+ "Boolean",
182
+ "RegExp",
183
+ "Set",
184
+ "Map",
185
+ "WeakSet",
186
+ "WeakMap",
187
+ "Symbol",
188
+ "Promise"
170
189
  ];
190
+ /**
191
+ * Creates a secure sandbox execution context with controlled globals
192
+ *
193
+ * @param options - Configuration for the sandbox
194
+ * @returns A VM context that can be used with executeCode()
195
+ *
196
+ * @example
197
+ * ```typescript
198
+ * const sandbox = createSandbox({
199
+ * globals: {
200
+ * cheerio: require('cheerio'),
201
+ * data: { foo: 'bar' }
202
+ * },
203
+ * timeout: 5000,
204
+ * allowedBuiltins: ['Array', 'Object', 'JSON']
205
+ * });
206
+ *
207
+ * const result = executeCode('data.foo', sandbox);
208
+ * // Returns: "bar"
209
+ * ```
210
+ */
171
211
  function createSandbox(options = {}) {
172
- const {
173
- globals = {},
174
- allowedBuiltins = DEFAULT_BUILTINS,
175
- allowConsole = false
176
- } = options;
177
- const sandbox = /* @__PURE__ */ Object.create(null);
178
- for (const builtin of allowedBuiltins) {
179
- if (builtin in globalThis) {
180
- sandbox[builtin] = globalThis[builtin];
181
- }
182
- }
183
- if (allowConsole) {
184
- sandbox.console = console;
185
- }
186
- Object.assign(sandbox, globals);
187
- return vm.createContext(sandbox);
212
+ const { globals = {}, allowedBuiltins = DEFAULT_BUILTINS, allowConsole = false } = options;
213
+ const sandbox = Object.create(null);
214
+ for (const builtin of allowedBuiltins) if (builtin in globalThis) sandbox[builtin] = globalThis[builtin];
215
+ if (allowConsole) sandbox.console = console;
216
+ Object.assign(sandbox, globals);
217
+ return vm.createContext(sandbox);
188
218
  }
219
+ /**
220
+ * Executes code in a sandbox with timeout and error handling
221
+ *
222
+ * @param code - The JavaScript code to execute
223
+ * @param sandbox - The VM context created by createSandbox()
224
+ * @param options - Execution options
225
+ * @returns The result of the code execution
226
+ * @throws {Error} If code execution fails or times out
227
+ *
228
+ * @example
229
+ * ```typescript
230
+ * const sandbox = createSandbox({
231
+ * globals: { x: 10, y: 20 }
232
+ * });
233
+ *
234
+ * const result = executeCode('x + y', sandbox);
235
+ * // Returns: 30
236
+ *
237
+ * // With a function
238
+ * const funcResult = executeCode(`
239
+ * function add(a, b) {
240
+ * return a + b;
241
+ * }
242
+ * add(x, y);
243
+ * `, sandbox);
244
+ * // Returns: 30
245
+ * ```
246
+ */
189
247
  function executeCode(code, sandbox, options = {}) {
190
- const {
191
- timeout = 5e3,
192
- filename = "generated-code.js",
193
- captureResult = true
194
- } = options;
195
- try {
196
- let wrappedCode;
197
- if (captureResult) {
198
- const hasMultipleStatements = code.includes(";") || code.includes("\n") && code.trim().split("\n").length > 1;
199
- const hasFunctionDef = /function\s+\w+|const\s+\w+\s*=\s*function|const\s+\w+\s*=\s*\(/i.test(
200
- code
201
- );
202
- if (hasMultipleStatements || hasFunctionDef) {
203
- wrappedCode = code;
204
- } else {
205
- wrappedCode = `(function() { return (${code}); })();`;
206
- }
207
- } else {
208
- wrappedCode = code;
209
- }
210
- const result = vm.runInContext(wrappedCode, sandbox, {
211
- timeout,
212
- filename,
213
- displayErrors: true
214
- });
215
- return result;
216
- } catch (error) {
217
- if (error instanceof Error) {
218
- const message = `Code execution failed: ${error.message}`;
219
- const enhancedError = new Error(message);
220
- enhancedError.stack = error.stack;
221
- throw enhancedError;
222
- }
223
- throw error;
224
- }
248
+ const { timeout = 5e3, filename = "generated-code.js", captureResult = true } = options;
249
+ try {
250
+ let wrappedCode;
251
+ if (captureResult) {
252
+ const hasMultipleStatements = code.includes(";") || code.includes("\n") && code.trim().split("\n").length > 1;
253
+ const hasFunctionDef = /function\s+\w+|const\s+\w+\s*=\s*function|const\s+\w+\s*=\s*\(/i.test(code);
254
+ if (hasMultipleStatements || hasFunctionDef) wrappedCode = code;
255
+ else wrappedCode = `(function() { return (${code}); })();`;
256
+ } else wrappedCode = code;
257
+ return vm.runInContext(wrappedCode, sandbox, {
258
+ timeout,
259
+ filename,
260
+ displayErrors: true
261
+ });
262
+ } catch (error) {
263
+ if (error instanceof Error) {
264
+ const message = `Code execution failed: ${error.message}`;
265
+ const enhancedError = new Error(message);
266
+ enhancedError.stack = error.stack;
267
+ throw enhancedError;
268
+ }
269
+ throw error;
270
+ }
225
271
  }
272
+ /**
273
+ * Executes async code in a sandbox with timeout and error handling
274
+ *
275
+ * @param code - The JavaScript code to execute (can contain async/await)
276
+ * @param sandbox - The VM context created by createSandbox()
277
+ * @param options - Execution options
278
+ * @returns Promise resolving to the result of the code execution
279
+ * @throws {Error} If code execution fails or times out
280
+ *
281
+ * @example
282
+ * ```typescript
283
+ * const sandbox = createSandbox({
284
+ * globals: {
285
+ * fetch: require('node-fetch')
286
+ * }
287
+ * });
288
+ *
289
+ * const result = await executeCodeAsync(`
290
+ * const response = await fetch('https://api.example.com/data');
291
+ * const data = await response.json();
292
+ * data;
293
+ * `, sandbox);
294
+ * ```
295
+ */
226
296
  async function executeCodeAsync(code, sandbox, options = {}) {
227
- const {
228
- timeout = 5e3,
229
- filename = "generated-code.js",
230
- captureResult = true
231
- } = options;
232
- try {
233
- let wrappedCode;
234
- if (captureResult) {
235
- const trimmedCode = code.trim();
236
- const lines = trimmedCode.split("\n");
237
- if (lines.length > 1 || trimmedCode.includes(";")) {
238
- const statements = trimmedCode.split("\n").filter((line) => line.trim());
239
- const lastLine = statements[statements.length - 1];
240
- const otherLines = statements.slice(0, -1);
241
- if (lastLine.trim().startsWith("return ")) {
242
- wrappedCode = `(async function() {
297
+ const { timeout = 5e3, filename = "generated-code.js", captureResult = true } = options;
298
+ try {
299
+ let wrappedCode;
300
+ if (captureResult) {
301
+ const trimmedCode = code.trim();
302
+ if (trimmedCode.split("\n").length > 1 || trimmedCode.includes(";")) {
303
+ const statements = trimmedCode.split("\n").filter((line) => line.trim());
304
+ const lastLine = statements[statements.length - 1];
305
+ const otherLines = statements.slice(0, -1);
306
+ if (lastLine.trim().startsWith("return ")) wrappedCode = `(async function() {
243
307
  ${trimmedCode}
244
308
  })();`;
245
- } else {
246
- const lastExpression = lastLine.trim().replace(/;$/, "");
247
- wrappedCode = `(async function() {
309
+ else {
310
+ const lastExpression = lastLine.trim().replace(/;$/, "");
311
+ wrappedCode = `(async function() {
248
312
  ${otherLines.join("\n")}
249
313
  return ${lastExpression};
250
314
  })();`;
251
- }
252
- } else {
253
- wrappedCode = `(async function() {
315
+ }
316
+ } else wrappedCode = `(async function() {
254
317
  return (${trimmedCode});
255
318
  })();`;
256
- }
257
- } else {
258
- wrappedCode = `(async function() {
319
+ } else wrappedCode = `(async function() {
259
320
  ${code}
260
321
  })();`;
261
- }
262
- const result = await vm.runInContext(wrappedCode, sandbox, {
263
- timeout,
264
- filename,
265
- displayErrors: true
266
- });
267
- return result;
268
- } catch (error) {
269
- if (error instanceof Error) {
270
- const message = `Async code execution failed: ${error.message}`;
271
- const enhancedError = new Error(message);
272
- enhancedError.stack = error.stack;
273
- throw enhancedError;
274
- }
275
- throw error;
276
- }
322
+ return await vm.runInContext(wrappedCode, sandbox, {
323
+ timeout,
324
+ filename,
325
+ displayErrors: true
326
+ });
327
+ } catch (error) {
328
+ if (error instanceof Error) {
329
+ const message = `Async code execution failed: ${error.message}`;
330
+ const enhancedError = new Error(message);
331
+ enhancedError.stack = error.stack;
332
+ throw enhancedError;
333
+ }
334
+ throw error;
335
+ }
277
336
  }
337
+ /**
338
+ * Convenience function to create a sandbox and execute code in one step
339
+ *
340
+ * @param code - The JavaScript code to execute
341
+ * @param options - Combined sandbox and execution options
342
+ * @returns The result of the code execution
343
+ *
344
+ * @example
345
+ * ```typescript
346
+ * const result = executeInSandbox('Math.sqrt(16)', {
347
+ * globals: { x: 10 },
348
+ * timeout: 1000
349
+ * });
350
+ * // Returns: 4
351
+ * ```
352
+ */
278
353
  function executeInSandbox(code, options = {}) {
279
- const sandbox = createSandbox(options);
280
- return executeCode(code, sandbox, options);
354
+ return executeCode(code, createSandbox(options), options);
281
355
  }
356
+ /**
357
+ * Convenience function to create a sandbox and execute async code in one step
358
+ *
359
+ * @param code - The JavaScript code to execute (can contain async/await)
360
+ * @param options - Combined sandbox and execution options
361
+ * @returns Promise resolving to the result of the code execution
362
+ *
363
+ * @example
364
+ * ```typescript
365
+ * const result = await executeInSandboxAsync(`
366
+ * const data = await Promise.resolve({ value: 42 });
367
+ * data.value;
368
+ * `, {
369
+ * timeout: 2000
370
+ * });
371
+ * // Returns: 42
372
+ * ```
373
+ */
282
374
  async function executeInSandboxAsync(code, options = {}) {
283
- const sandbox = createSandbox(options);
284
- return executeCodeAsync(code, sandbox, options);
375
+ return executeCodeAsync(code, createSandbox(options), options);
285
376
  }
377
+ //#endregion
378
+ //#region src/web.ts
379
+ /**
380
+ * Web and URL utility functions
381
+ *
382
+ * General-purpose utilities for working with URLs and web content.
383
+ * Used by scrapers, note systems, and content processors.
384
+ */
385
+ /**
386
+ * Normalize URL for consistent key storage
387
+ * Removes tracking params, sorts query string, lowercases, etc.
388
+ *
389
+ * @param url - The URL to normalize
390
+ * @returns Normalized URL string
391
+ */
286
392
  function normalizeUrl(url) {
287
- const parsed = new URL(url);
288
- parsed.protocol = parsed.protocol.toLowerCase();
289
- parsed.hostname = parsed.hostname.toLowerCase();
290
- parsed.hostname = parsed.hostname.replace(/^www\./, "");
291
- if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") {
292
- parsed.port = "";
293
- }
294
- parsed.hash = "";
295
- const params = new URLSearchParams(parsed.search);
296
- const filtered = new URLSearchParams();
297
- const trackingParams = [
298
- "utm_source",
299
- "utm_medium",
300
- "utm_campaign",
301
- "utm_content",
302
- "utm_term",
303
- "fbclid",
304
- "gclid",
305
- "msclkid",
306
- "_ga",
307
- "mc_cid",
308
- "mc_eid"
309
- ];
310
- Array.from(params.keys()).sort().forEach((key) => {
311
- if (!trackingParams.includes(key)) {
312
- filtered.set(key, params.get(key));
313
- }
314
- });
315
- parsed.search = filtered.toString();
316
- return parsed.toString();
393
+ const parsed = new URL(url);
394
+ parsed.protocol = parsed.protocol.toLowerCase();
395
+ parsed.hostname = parsed.hostname.toLowerCase();
396
+ parsed.hostname = parsed.hostname.replace(/^www\./, "");
397
+ if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") parsed.port = "";
398
+ parsed.hash = "";
399
+ const params = new URLSearchParams(parsed.search);
400
+ const filtered = new URLSearchParams();
401
+ const trackingParams = [
402
+ "utm_source",
403
+ "utm_medium",
404
+ "utm_campaign",
405
+ "utm_content",
406
+ "utm_term",
407
+ "fbclid",
408
+ "gclid",
409
+ "msclkid",
410
+ "_ga",
411
+ "mc_cid",
412
+ "mc_eid"
413
+ ];
414
+ Array.from(params.keys()).sort().forEach((key) => {
415
+ if (!trackingParams.includes(key)) filtered.set(key, params.get(key));
416
+ });
417
+ parsed.search = filtered.toString();
418
+ return parsed.toString();
317
419
  }
420
+ /**
421
+ * Generate hierarchical scope from URL for organized note storage
422
+ *
423
+ * @param url - The URL to generate scope from
424
+ * @param baseScope - Base scope prefix (default: 'discovery/parser')
425
+ * @returns Hierarchical scope string
426
+ *
427
+ * @example
428
+ * generateScopeFromUrl('https://cityofboston.gov/meetings/minutes')
429
+ * // Returns: 'discovery/parser/cityofboston.gov/meetings'
430
+ */
318
431
  function generateScopeFromUrl(url, baseScope = "discovery/parser") {
319
- const parsed = new URL(normalizeUrl(url));
320
- const domain = parsed.hostname;
321
- const pathParts = parsed.pathname.split("/").filter((p2) => p2);
322
- const pageType = pathParts[0] || "index";
323
- return `${baseScope}/${domain}/${pageType}`;
432
+ const parsed = new URL(normalizeUrl(url));
433
+ return `${baseScope}/${parsed.hostname}/${parsed.pathname.split("/").filter((p) => p)[0] || "index"}`;
324
434
  }
435
+ /**
436
+ * Hash page content for change detection
437
+ * Uses SHA-256 to create a unique fingerprint of the HTML
438
+ *
439
+ * @param html - Page HTML content
440
+ * @returns SHA-256 hash as hex string
441
+ */
325
442
  function hashPageContent(html) {
326
- return createHash("sha256").update(html).digest("hex");
443
+ return createHash("sha256").update(html).digest("hex");
327
444
  }
328
- const PACKAGE_VERSION_INITIALIZED = true;
329
- export {
330
- A as ApiError,
331
- B as BaseError,
332
- D as DatabaseError,
333
- E as ErrorCode,
334
- F as FileError,
335
- N as NetworkError,
336
- PACKAGE_VERSION_INITIALIZED,
337
- P as ParsingError,
338
- T as TimeoutError,
339
- V as ValidationError,
340
- a as addInterval,
341
- c as camelCase,
342
- b as convertType,
343
- d as createId,
344
- createSandbox,
345
- e as dateInString,
346
- f as disableLogging,
347
- g as domainToCamel,
348
- h as enableLogging,
349
- executeCode,
350
- executeCodeAsync,
351
- executeInSandbox,
352
- executeInSandboxAsync,
353
- i as extractAllCodeBlocks,
354
- j as extractCodeBlock,
355
- k as extractFunctionDefinition,
356
- l as extractJSON,
357
- m as formatDate,
358
- generateScopeFromUrl,
359
- n as getLogger,
360
- o as getTempDirectory,
361
- hashPageContent,
362
- p as isArray,
363
- isCuid,
364
- q as isPlainObject,
365
- r as isPlural,
366
- s as isSafeCode,
367
- t as isSingular,
368
- u as isUrl,
369
- v as isValidDate,
370
- w as keysToCamel,
371
- x as keysToSnake,
372
- y as loadEnvConfig,
373
- z as logTicker,
374
- C as makeId,
375
- G as makeSlug,
376
- normalizeUrl,
377
- H as parseAmazonDateString,
378
- parseCliArgs,
379
- I as parseDate,
380
- parseFlagArgs,
381
- J as pluralizeWord,
382
- K as prettyDate,
383
- L as setLogger,
384
- M as singularize,
385
- O as sleep,
386
- Q as snakeCase,
387
- R as toCamelCase,
388
- S as toScreamingSnakeCase,
389
- U as urlFilename,
390
- W as urlPath,
391
- X as validateCode,
392
- Y as waitFor
393
- };
394
- //# sourceMappingURL=index.js.map
445
+ //#endregion
446
+ //#region src/index.ts
447
+ /** @internal */
448
+ var PACKAGE_VERSION_INITIALIZED = true;
449
+ //#endregion
450
+ export { ApiError, BaseError, DatabaseError, ErrorCode, FileError, NetworkError, PACKAGE_VERSION_INITIALIZED, ParsingError, TimeoutError, ValidationError, addInterval, camelCase, convertType, createId, createSandbox, dateInString, disableLogging, domainToCamel, enableLogging, executeCode, executeCodeAsync, executeInSandbox, executeInSandboxAsync, extractAllCodeBlocks, extractCodeBlock, extractFunctionDefinition, extractJSON, formatDate, generateScopeFromUrl, getLogger, getTempDirectory, hashPageContent, isArray, isCuid, isPlainObject, isPlural, isSafeCode, isSingular, isUrl, isValidDate, keysToCamel, keysToSnake, loadEnvConfig, logTicker, makeId, makeSlug, normalizeUrl, parseAmazonDateString, parseCliArgs, parseDate, parseFlagArgs, pluralizeWord, prettyDate, setLogger, singularize, sleep, snakeCase, toCamelCase, toScreamingSnakeCase, urlFilename, urlPath, validateCode, waitFor };
451
+
452
+ //# sourceMappingURL=index.js.map