@memberjunction/core-actions 2.54.0 → 2.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/custom/ai/execute-ai-prompt.action.d.ts +94 -0
- package/dist/custom/ai/execute-ai-prompt.action.d.ts.map +1 -0
- package/dist/custom/ai/execute-ai-prompt.action.js +274 -0
- package/dist/custom/ai/execute-ai-prompt.action.js.map +1 -0
- package/dist/custom/communication/slack-webhook.action.d.ts +101 -0
- package/dist/custom/communication/slack-webhook.action.d.ts.map +1 -0
- package/dist/custom/communication/slack-webhook.action.js +269 -0
- package/dist/custom/communication/slack-webhook.action.js.map +1 -0
- package/dist/custom/communication/teams-webhook.action.d.ts +96 -0
- package/dist/custom/communication/teams-webhook.action.d.ts.map +1 -0
- package/dist/custom/communication/teams-webhook.action.js +275 -0
- package/dist/custom/communication/teams-webhook.action.js.map +1 -0
- package/dist/custom/data/aggregate-data.action.d.ts +91 -0
- package/dist/custom/data/aggregate-data.action.d.ts.map +1 -0
- package/dist/custom/data/aggregate-data.action.js +333 -0
- package/dist/custom/data/aggregate-data.action.js.map +1 -0
- package/dist/custom/data/csv-parser.action.d.ts +68 -0
- package/dist/custom/data/csv-parser.action.d.ts.map +1 -0
- package/dist/custom/data/csv-parser.action.js +184 -0
- package/dist/custom/data/csv-parser.action.js.map +1 -0
- package/dist/custom/data/data-mapper.action.d.ts +89 -0
- package/dist/custom/data/data-mapper.action.d.ts.map +1 -0
- package/dist/custom/data/data-mapper.action.js +325 -0
- package/dist/custom/data/data-mapper.action.js.map +1 -0
- package/dist/custom/data/json-transform.action.d.ts +57 -0
- package/dist/custom/data/json-transform.action.d.ts.map +1 -0
- package/dist/custom/data/json-transform.action.js +205 -0
- package/dist/custom/data/json-transform.action.js.map +1 -0
- package/dist/custom/data/xml-parser.action.d.ts +85 -0
- package/dist/custom/data/xml-parser.action.d.ts.map +1 -0
- package/dist/custom/data/xml-parser.action.js +326 -0
- package/dist/custom/data/xml-parser.action.js.map +1 -0
- package/dist/custom/files/excel-reader.action.d.ts +86 -0
- package/dist/custom/files/excel-reader.action.d.ts.map +1 -0
- package/dist/custom/files/excel-reader.action.js +348 -0
- package/dist/custom/files/excel-reader.action.js.map +1 -0
- package/dist/custom/files/excel-writer.action.d.ts +79 -0
- package/dist/custom/files/excel-writer.action.d.ts.map +1 -0
- package/dist/custom/files/excel-writer.action.js +325 -0
- package/dist/custom/files/excel-writer.action.js.map +1 -0
- package/dist/custom/files/file-compress.action.d.ts +90 -0
- package/dist/custom/files/file-compress.action.d.ts.map +1 -0
- package/dist/custom/files/file-compress.action.js +346 -0
- package/dist/custom/files/file-compress.action.js.map +1 -0
- package/dist/custom/files/pdf-extractor.action.d.ts +86 -0
- package/dist/custom/files/pdf-extractor.action.d.ts.map +1 -0
- package/dist/custom/files/pdf-extractor.action.js +247 -0
- package/dist/custom/files/pdf-extractor.action.js.map +1 -0
- package/dist/custom/files/pdf-generator.action.d.ts +70 -0
- package/dist/custom/files/pdf-generator.action.d.ts.map +1 -0
- package/dist/custom/files/pdf-generator.action.js +286 -0
- package/dist/custom/files/pdf-generator.action.js.map +1 -0
- package/dist/custom/integration/api-rate-limiter.action.d.ts +88 -0
- package/dist/custom/integration/api-rate-limiter.action.d.ts.map +1 -0
- package/dist/custom/integration/api-rate-limiter.action.js +363 -0
- package/dist/custom/integration/api-rate-limiter.action.js.map +1 -0
- package/dist/custom/integration/graphql-query.action.d.ts +92 -0
- package/dist/custom/integration/graphql-query.action.d.ts.map +1 -0
- package/dist/custom/integration/graphql-query.action.js +251 -0
- package/dist/custom/integration/graphql-query.action.js.map +1 -0
- package/dist/custom/integration/http-request.action.d.ts +96 -0
- package/dist/custom/integration/http-request.action.d.ts.map +1 -0
- package/dist/custom/integration/http-request.action.js +316 -0
- package/dist/custom/integration/http-request.action.js.map +1 -0
- package/dist/custom/integration/oauth-flow.action.d.ts +130 -0
- package/dist/custom/integration/oauth-flow.action.d.ts.map +1 -0
- package/dist/custom/integration/oauth-flow.action.js +624 -0
- package/dist/custom/integration/oauth-flow.action.js.map +1 -0
- package/dist/custom/security/password-strength.action.d.ts +129 -0
- package/dist/custom/security/password-strength.action.d.ts.map +1 -0
- package/dist/custom/security/password-strength.action.js +463 -0
- package/dist/custom/security/password-strength.action.js.map +1 -0
- package/dist/custom/utilities/base-file-handler.d.ts +54 -0
- package/dist/custom/utilities/base-file-handler.d.ts.map +1 -0
- package/dist/custom/utilities/base-file-handler.js +167 -0
- package/dist/custom/utilities/base-file-handler.js.map +1 -0
- package/dist/custom/utilities/index.d.ts +3 -0
- package/dist/custom/utilities/index.d.ts.map +1 -0
- package/dist/custom/utilities/index.js +19 -0
- package/dist/custom/utilities/index.js.map +1 -0
- package/dist/custom/utilities/json-param-helper.d.ts +33 -0
- package/dist/custom/utilities/json-param-helper.d.ts.map +1 -0
- package/dist/custom/utilities/json-param-helper.js +66 -0
- package/dist/custom/utilities/json-param-helper.js.map +1 -0
- package/dist/custom/web/duckduckgo-rate-limiter.d.ts +72 -0
- package/dist/custom/web/duckduckgo-rate-limiter.d.ts.map +1 -0
- package/dist/custom/web/duckduckgo-rate-limiter.js +175 -0
- package/dist/custom/web/duckduckgo-rate-limiter.js.map +1 -0
- package/dist/custom/web/web-search.action.d.ts +1 -0
- package/dist/custom/web/web-search.action.d.ts.map +1 -1
- package/dist/custom/web/web-search.action.js +48 -20
- package/dist/custom/web/web-search.action.js.map +1 -1
- package/dist/custom/workflow/conditional.action.d.ts +94 -0
- package/dist/custom/workflow/conditional.action.d.ts.map +1 -0
- package/dist/custom/workflow/conditional.action.js +274 -0
- package/dist/custom/workflow/conditional.action.js.map +1 -0
- package/dist/custom/workflow/delay.action.d.ts +76 -0
- package/dist/custom/workflow/delay.action.d.ts.map +1 -0
- package/dist/custom/workflow/delay.action.js +187 -0
- package/dist/custom/workflow/delay.action.js.map +1 -0
- package/dist/custom/workflow/loop.action.d.ts +112 -0
- package/dist/custom/workflow/loop.action.d.ts.map +1 -0
- package/dist/custom/workflow/loop.action.js +380 -0
- package/dist/custom/workflow/loop.action.js.map +1 -0
- package/dist/custom/workflow/parallel-execute.action.d.ts +105 -0
- package/dist/custom/workflow/parallel-execute.action.d.ts.map +1 -0
- package/dist/custom/workflow/parallel-execute.action.js +457 -0
- package/dist/custom/workflow/parallel-execute.action.js.map +1 -0
- package/dist/custom/workflow/retry.action.d.ts +90 -0
- package/dist/custom/workflow/retry.action.d.ts.map +1 -0
- package/dist/custom/workflow/retry.action.js +378 -0
- package/dist/custom/workflow/retry.action.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +75 -0
- package/dist/index.js.map +1 -1
- package/package.json +25 -10
- package/readme.md +11270 -101
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BaseFileHandlerAction = void 0;
|
|
4
|
+
const actions_1 = require("@memberjunction/actions");
|
|
5
|
+
const core_1 = require("@memberjunction/core");
|
|
6
|
+
/**
|
|
7
|
+
* Base class for actions that handle file inputs from multiple sources
|
|
8
|
+
* Provides common functionality for loading files from:
|
|
9
|
+
* - MJ Storage (Document Libraries)
|
|
10
|
+
* - URLs
|
|
11
|
+
* - Direct data
|
|
12
|
+
*/
|
|
13
|
+
class BaseFileHandlerAction extends actions_1.BaseAction {
|
|
14
|
+
/**
|
|
15
|
+
* Get file content from various sources based on parameters
|
|
16
|
+
* Priority: FileID > FileURL > Data parameter
|
|
17
|
+
*
|
|
18
|
+
* @param params - Action parameters
|
|
19
|
+
* @param dataParamName - Name of the parameter containing direct data
|
|
20
|
+
* @param fileParamName - Name of the parameter containing file ID (default: 'FileID')
|
|
21
|
+
* @param urlParamName - Name of the parameter containing file URL (default: 'FileURL')
|
|
22
|
+
* @returns Object with content and metadata
|
|
23
|
+
*/
|
|
24
|
+
async getFileContent(params, dataParamName, fileParamName = 'FileID', urlParamName = 'FileURL') {
|
|
25
|
+
// Check for FileID first (MJ Storage)
|
|
26
|
+
const fileIdParam = params.Params.find(p => p.Name.trim().toLowerCase() === fileParamName.toLowerCase());
|
|
27
|
+
if (fileIdParam?.Value) {
|
|
28
|
+
return await this.loadFromMJStorage(fileIdParam.Value.toString(), params);
|
|
29
|
+
}
|
|
30
|
+
// Check for FileURL
|
|
31
|
+
const fileUrlParam = params.Params.find(p => p.Name.trim().toLowerCase() === urlParamName.toLowerCase());
|
|
32
|
+
if (fileUrlParam?.Value) {
|
|
33
|
+
return await this.loadFromURL(fileUrlParam.Value.toString());
|
|
34
|
+
}
|
|
35
|
+
// Check for direct data
|
|
36
|
+
const dataParam = params.Params.find(p => p.Name.trim().toLowerCase() === dataParamName.toLowerCase());
|
|
37
|
+
if (dataParam?.Value) {
|
|
38
|
+
return {
|
|
39
|
+
content: dataParam.Value.toString(),
|
|
40
|
+
source: 'direct'
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
throw new Error(`No input provided. Please provide ${fileParamName}, ${urlParamName}, or ${dataParamName}`);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Load file from MJ Storage (Document Libraries)
|
|
47
|
+
*/
|
|
48
|
+
async loadFromMJStorage(fileId, params) {
|
|
49
|
+
try {
|
|
50
|
+
const rv = new core_1.RunView();
|
|
51
|
+
const result = await rv.RunView({
|
|
52
|
+
EntityName: 'Document Libraries',
|
|
53
|
+
ExtraFilter: `ID = '${fileId}'`,
|
|
54
|
+
ResultType: 'entity_object'
|
|
55
|
+
}, params.ContextUser);
|
|
56
|
+
if (!result.Success || !result.Results || result.Results.length === 0) {
|
|
57
|
+
throw new Error(`File not found in Document Libraries: ${fileId}`);
|
|
58
|
+
}
|
|
59
|
+
const doc = result.Results[0];
|
|
60
|
+
// Get the actual file content (this would need implementation based on your storage provider)
|
|
61
|
+
// For now, returning the file info - actual implementation would fetch from storage
|
|
62
|
+
return {
|
|
63
|
+
content: '', // TODO: Implement actual file retrieval based on storage provider
|
|
64
|
+
fileName: doc.Get('FileName'),
|
|
65
|
+
mimeType: doc.Get('MimeType'),
|
|
66
|
+
source: 'storage'
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
throw new Error(`Failed to load file from storage: ${error instanceof Error ? error.message : String(error)}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Load file from URL
|
|
75
|
+
*/
|
|
76
|
+
async loadFromURL(url) {
|
|
77
|
+
try {
|
|
78
|
+
// Validate URL
|
|
79
|
+
const urlObj = new URL(url);
|
|
80
|
+
if (!['http:', 'https:'].includes(urlObj.protocol)) {
|
|
81
|
+
throw new Error('Only HTTP and HTTPS URLs are supported');
|
|
82
|
+
}
|
|
83
|
+
const response = await fetch(url);
|
|
84
|
+
if (!response.ok) {
|
|
85
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
86
|
+
}
|
|
87
|
+
const contentType = response.headers.get('content-type');
|
|
88
|
+
const contentDisposition = response.headers.get('content-disposition');
|
|
89
|
+
let fileName;
|
|
90
|
+
// Extract filename from content-disposition if available
|
|
91
|
+
if (contentDisposition) {
|
|
92
|
+
const match = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
|
|
93
|
+
if (match && match[1]) {
|
|
94
|
+
fileName = match[1].replace(/['"]/g, '');
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
// If no filename from header, extract from URL
|
|
98
|
+
if (!fileName) {
|
|
99
|
+
fileName = urlObj.pathname.split('/').pop();
|
|
100
|
+
}
|
|
101
|
+
const content = await response.text();
|
|
102
|
+
return {
|
|
103
|
+
content,
|
|
104
|
+
fileName,
|
|
105
|
+
mimeType: contentType || undefined,
|
|
106
|
+
source: 'url'
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
throw new Error(`Failed to load file from URL: ${error instanceof Error ? error.message : String(error)}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Save file to MJ Storage
|
|
115
|
+
*/
|
|
116
|
+
async saveToMJStorage(content, fileName, mimeType, params) {
|
|
117
|
+
try {
|
|
118
|
+
const md = new core_1.Metadata();
|
|
119
|
+
const doc = await md.GetEntityObject('Document Libraries', params.ContextUser);
|
|
120
|
+
if (!doc) {
|
|
121
|
+
throw new Error('Failed to create Document Library entity');
|
|
122
|
+
}
|
|
123
|
+
doc.Set('FileName', fileName);
|
|
124
|
+
doc.Set('MimeType', mimeType);
|
|
125
|
+
// TODO: Set appropriate fields based on your Document Libraries schema
|
|
126
|
+
// doc.Set('FileSize', Buffer.byteLength(content));
|
|
127
|
+
// doc.Set('StorageProvider', 'default');
|
|
128
|
+
const saveResult = await doc.Save();
|
|
129
|
+
if (!saveResult) {
|
|
130
|
+
throw new Error('Failed to save document to library');
|
|
131
|
+
}
|
|
132
|
+
// TODO: Actually store the file content to the storage provider
|
|
133
|
+
return doc.Get('ID');
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
throw new Error(`Failed to save file to storage: ${error instanceof Error ? error.message : String(error)}`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Extract parameter value by name (case-insensitive)
|
|
141
|
+
*/
|
|
142
|
+
getParamValue(params, paramName) {
|
|
143
|
+
const param = params.Params.find(p => p.Name.trim().toLowerCase() === paramName.toLowerCase());
|
|
144
|
+
return param?.Value;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Get boolean parameter value with default
|
|
148
|
+
*/
|
|
149
|
+
getBooleanParam(params, paramName, defaultValue = false) {
|
|
150
|
+
const value = this.getParamValue(params, paramName);
|
|
151
|
+
if (value === undefined || value === null)
|
|
152
|
+
return defaultValue;
|
|
153
|
+
return String(value).toLowerCase() === 'true';
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Get numeric parameter value with default
|
|
157
|
+
*/
|
|
158
|
+
getNumericParam(params, paramName, defaultValue = 0) {
|
|
159
|
+
const value = this.getParamValue(params, paramName);
|
|
160
|
+
if (value === undefined || value === null)
|
|
161
|
+
return defaultValue;
|
|
162
|
+
const num = Number(value);
|
|
163
|
+
return isNaN(num) ? defaultValue : num;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
exports.BaseFileHandlerAction = BaseFileHandlerAction;
|
|
167
|
+
//# sourceMappingURL=base-file-handler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-file-handler.js","sourceRoot":"","sources":["../../../src/custom/utilities/base-file-handler.ts"],"names":[],"mappings":";;;AAAA,qDAAqD;AAErD,+CAAqE;AAErE;;;;;;GAMG;AACH,MAAsB,qBAAsB,SAAQ,oBAAU;IAC1D;;;;;;;;;OASG;IACO,KAAK,CAAC,cAAc,CAC1B,MAAuB,EACvB,aAAqB,EACrB,gBAAwB,QAAQ,EAChC,eAAuB,SAAS;QAOhC,sCAAsC;QACtC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,IAAI,WAAW,EAAE,KAAK,EAAE,CAAC;YACrB,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;QAC9E,CAAC;QAED,oBAAoB;QACpB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,IAAI,YAAY,EAAE,KAAK,EAAE,CAAC;YACtB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;QACvG,IAAI,SAAS,EAAE,KAAK,EAAE,CAAC;YACnB,OAAO;gBACH,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE;gBACnC,MAAM,EAAE,QAAQ;aACnB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,aAAa,KAAK,YAAY,QAAQ,aAAa,EAAE,CAAC,CAAC;IAChH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAC,MAAc,EAAE,MAAuB;QAMnE,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAa;gBACxC,UAAU,EAAE,oBAAoB;gBAChC,WAAW,EAAE,SAAS,MAAM,GAAG;gBAC/B,UAAU,EAAE,eAAe;aAC9B,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAEvB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAE9B,8FAA8F;YAC9F,oFAAoF;YACpF,OAAO;gBACH,OAAO,EAAE,EAAE,EAAE,kEAAkE;gBAC/E,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC;gBAC7B,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC;gBAC7B,MAAM,EAAE,SAAS;aACpB,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnH,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,GAAW;QAMjC,IAAI,CAAC;YACD,eAAe;YACf,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACzD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACvE,IAAI,QAA4B,CAAC;YAEjC,yDAAyD;YACzD,IAAI,kBAAkB,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBACjF,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpB,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAChD,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,OAAO;gBACH,OAAO;gBACP,QAAQ;gBACR,QAAQ,EAAE,WAAW,IAAI,SAAS;gBAClC,MAAM,EAAE,KAAK;aAChB,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,eAAe,CAC3B,OAAwB,EACxB,QAAgB,EAChB,QAAgB,EAChB,MAAuB;QAEvB,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,eAAe,CAAa,oBAAoB,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAE3F,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAChE,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC9B,uEAAuE;YACvE,mDAAmD;YACnD,yCAAyC;YAEzC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YAC1D,CAAC;YAED,gEAAgE;YAEhE,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;IACL,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,MAAuB,EAAE,SAAiB;QAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/F,OAAO,KAAK,EAAE,KAAK,CAAC;IACxB,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAuB,EAAE,SAAiB,EAAE,eAAwB,KAAK;QAC/F,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QAC/D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IAClD,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAuB,EAAE,SAAiB,EAAE,eAAuB,CAAC;QAC1F,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3C,CAAC;CACJ;AAnMD,sDAmMC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/custom/utilities/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./base-file-handler"), exports);
|
|
18
|
+
__exportStar(require("./json-param-helper"), exports);
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/custom/utilities/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,sDAAoC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { RunActionParams } from "@memberjunction/actions-base";
|
|
2
|
+
/**
|
|
3
|
+
* Helper class for handling JSON parameters that can be either objects or strings
|
|
4
|
+
*/
|
|
5
|
+
export declare class JSONParamHelper {
|
|
6
|
+
/**
|
|
7
|
+
* Get a parameter value that can be either a JSON object or a JSON string
|
|
8
|
+
* First checks for the object parameter, then falls back to the string parameter
|
|
9
|
+
*
|
|
10
|
+
* @param params - The action parameters
|
|
11
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
12
|
+
* @returns The parsed value or undefined
|
|
13
|
+
*/
|
|
14
|
+
static getJSONParam(params: RunActionParams, paramName: string): any;
|
|
15
|
+
/**
|
|
16
|
+
* Check if a JSON parameter exists (either as object or string)
|
|
17
|
+
*
|
|
18
|
+
* @param params - The action parameters
|
|
19
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
20
|
+
* @returns True if the parameter exists
|
|
21
|
+
*/
|
|
22
|
+
static hasJSONParam(params: RunActionParams, paramName: string): boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Get a required JSON parameter, throwing an error if not found
|
|
25
|
+
*
|
|
26
|
+
* @param params - The action parameters
|
|
27
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
28
|
+
* @returns The parsed value
|
|
29
|
+
* @throws Error if parameter is not found
|
|
30
|
+
*/
|
|
31
|
+
static getRequiredJSONParam(params: RunActionParams, paramName: string): any;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=json-param-helper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-param-helper.d.ts","sourceRoot":"","sources":["../../../src/custom/utilities/json-param-helper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAE/D;;GAEG;AACH,qBAAa,eAAe;IACxB;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG;IAoBpE;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO;IAUxE;;;;;;;OAOG;IACH,MAAM,CAAC,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG;CAO/E"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JSONParamHelper = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Helper class for handling JSON parameters that can be either objects or strings
|
|
6
|
+
*/
|
|
7
|
+
class JSONParamHelper {
|
|
8
|
+
/**
|
|
9
|
+
* Get a parameter value that can be either a JSON object or a JSON string
|
|
10
|
+
* First checks for the object parameter, then falls back to the string parameter
|
|
11
|
+
*
|
|
12
|
+
* @param params - The action parameters
|
|
13
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
14
|
+
* @returns The parsed value or undefined
|
|
15
|
+
*/
|
|
16
|
+
static getJSONParam(params, paramName) {
|
|
17
|
+
// First check for the object parameter
|
|
18
|
+
const objectParam = params.Params.find(p => p.Name.trim().toLowerCase() === paramName.toLowerCase());
|
|
19
|
+
if (objectParam?.Value !== undefined && objectParam?.Value !== null) {
|
|
20
|
+
return objectParam.Value;
|
|
21
|
+
}
|
|
22
|
+
// Then check for the string parameter
|
|
23
|
+
const stringParam = params.Params.find(p => p.Name.trim().toLowerCase() === `${paramName}string`.toLowerCase());
|
|
24
|
+
if (stringParam?.Value !== undefined && stringParam?.Value !== null) {
|
|
25
|
+
try {
|
|
26
|
+
return JSON.parse(stringParam.Value.toString());
|
|
27
|
+
}
|
|
28
|
+
catch (e) {
|
|
29
|
+
throw new Error(`Failed to parse ${paramName}String parameter as JSON: ${e instanceof Error ? e.message : String(e)}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Check if a JSON parameter exists (either as object or string)
|
|
36
|
+
*
|
|
37
|
+
* @param params - The action parameters
|
|
38
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
39
|
+
* @returns True if the parameter exists
|
|
40
|
+
*/
|
|
41
|
+
static hasJSONParam(params, paramName) {
|
|
42
|
+
const objectParam = params.Params.find(p => p.Name.trim().toLowerCase() === paramName.toLowerCase());
|
|
43
|
+
if (objectParam?.Value !== undefined && objectParam?.Value !== null) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
const stringParam = params.Params.find(p => p.Name.trim().toLowerCase() === `${paramName}string`.toLowerCase());
|
|
47
|
+
return stringParam?.Value !== undefined && stringParam?.Value !== null;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Get a required JSON parameter, throwing an error if not found
|
|
51
|
+
*
|
|
52
|
+
* @param params - The action parameters
|
|
53
|
+
* @param paramName - The base parameter name (without String suffix)
|
|
54
|
+
* @returns The parsed value
|
|
55
|
+
* @throws Error if parameter is not found
|
|
56
|
+
*/
|
|
57
|
+
static getRequiredJSONParam(params, paramName) {
|
|
58
|
+
const value = this.getJSONParam(params, paramName);
|
|
59
|
+
if (value === undefined) {
|
|
60
|
+
throw new Error(`${paramName} parameter is required (can be provided as ${paramName} object or ${paramName}String)`);
|
|
61
|
+
}
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
exports.JSONParamHelper = JSONParamHelper;
|
|
66
|
+
//# sourceMappingURL=json-param-helper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-param-helper.js","sourceRoot":"","sources":["../../../src/custom/utilities/json-param-helper.ts"],"names":[],"mappings":";;;AAEA;;GAEG;AACH,MAAa,eAAe;IACxB;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,MAAuB,EAAE,SAAiB;QAC1D,uCAAuC;QACvC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QACrG,IAAI,WAAW,EAAE,KAAK,KAAK,SAAS,IAAI,WAAW,EAAE,KAAK,KAAK,IAAI,EAAE,CAAC;YAClE,OAAO,WAAW,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,GAAG,SAAS,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAChH,IAAI,WAAW,EAAE,KAAK,KAAK,SAAS,IAAI,WAAW,EAAE,KAAK,KAAK,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC;gBACD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,mBAAmB,SAAS,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3H,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,MAAuB,EAAE,SAAiB;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QACrG,IAAI,WAAW,EAAE,KAAK,KAAK,SAAS,IAAI,WAAW,EAAE,KAAK,KAAK,IAAI,EAAE,CAAC;YAClE,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,GAAG,SAAS,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAChH,OAAO,WAAW,EAAE,KAAK,KAAK,SAAS,IAAI,WAAW,EAAE,KAAK,KAAK,IAAI,CAAC;IAC3E,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,oBAAoB,CAAC,MAAuB,EAAE,SAAiB;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACnD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,8CAA8C,SAAS,cAAc,SAAS,SAAS,CAAC,CAAC;QACzH,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ;AA7DD,0CA6DC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export interface RateLimitConfig {
|
|
2
|
+
maxConcurrent: number;
|
|
3
|
+
queueTimeout: number;
|
|
4
|
+
retryDelay: number;
|
|
5
|
+
maxRetries: number;
|
|
6
|
+
resetQueueAfterMs: number;
|
|
7
|
+
}
|
|
8
|
+
export interface SearchRequest {
|
|
9
|
+
url: string;
|
|
10
|
+
options: RequestInit;
|
|
11
|
+
disableQueueing?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface SearchResponse {
|
|
14
|
+
response: Response;
|
|
15
|
+
text: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Adaptive rate limiter for DuckDuckGo searches
|
|
19
|
+
* - Starts with immediate execution (no queue)
|
|
20
|
+
* - Activates queue on rate limit errors (202, 403, 429)
|
|
21
|
+
* - Automatically deactivates queue after success period
|
|
22
|
+
*/
|
|
23
|
+
export declare class DuckDuckGoRateLimiter {
|
|
24
|
+
private requestQueue$;
|
|
25
|
+
private isQueueActive;
|
|
26
|
+
private lastRateLimitTime;
|
|
27
|
+
private resetQueueTimer?;
|
|
28
|
+
private activeRequests;
|
|
29
|
+
private readonly config;
|
|
30
|
+
private readonly rateLimitStatusCodes;
|
|
31
|
+
constructor(config?: Partial<RateLimitConfig>);
|
|
32
|
+
/**
|
|
33
|
+
* Process requests through the queue when active
|
|
34
|
+
*/
|
|
35
|
+
private setupQueueProcessor;
|
|
36
|
+
/**
|
|
37
|
+
* Execute a search request with retry logic
|
|
38
|
+
*/
|
|
39
|
+
private executeRequest;
|
|
40
|
+
/**
|
|
41
|
+
* Activate the request queue due to rate limiting
|
|
42
|
+
*/
|
|
43
|
+
private activateQueue;
|
|
44
|
+
/**
|
|
45
|
+
* Schedule queue deactivation after successful requests
|
|
46
|
+
*/
|
|
47
|
+
private scheduleQueueReset;
|
|
48
|
+
/**
|
|
49
|
+
* Execute a search request through the rate limiter
|
|
50
|
+
*/
|
|
51
|
+
search(request: SearchRequest): Promise<SearchResponse>;
|
|
52
|
+
/**
|
|
53
|
+
* Get current queue status
|
|
54
|
+
*/
|
|
55
|
+
getStatus(): {
|
|
56
|
+
isQueueActive: boolean;
|
|
57
|
+
activeRequests: number;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* Force queue activation (for testing)
|
|
61
|
+
*/
|
|
62
|
+
forceActivateQueue(): void;
|
|
63
|
+
/**
|
|
64
|
+
* Force queue deactivation (for testing)
|
|
65
|
+
*/
|
|
66
|
+
forceDeactivateQueue(): void;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Get the singleton rate limiter instance
|
|
70
|
+
*/
|
|
71
|
+
export declare function getDuckDuckGoRateLimiter(config?: Partial<RateLimitConfig>): DuckDuckGoRateLimiter;
|
|
72
|
+
//# sourceMappingURL=duckduckgo-rate-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckduckgo-rate-limiter.d.ts","sourceRoot":"","sources":["../../../src/custom/web/duckduckgo-rate-limiter.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,eAAe;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,WAAW,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC3B,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAC9B,OAAO,CAAC,aAAa,CAIhB;IAEL,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,cAAc,CAAK;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAMrB;IAEF,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAmB;gBAE5C,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;IAK7C;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA8B3B;;OAEG;IACH,OAAO,CAAC,cAAc;IAuCtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAcrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAkB1B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC;IAiB7D;;OAEG;IACH,SAAS,IAAI;QAAE,aAAa,EAAE,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE;IAO/D;;OAEG;IACH,kBAAkB,IAAI,IAAI;IAI1B;;OAEG;IACH,oBAAoB,IAAI,IAAI;CAO/B;AAKD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,qBAAqB,CAKjG"}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getDuckDuckGoRateLimiter = exports.DuckDuckGoRateLimiter = void 0;
|
|
4
|
+
const rxjs_1 = require("rxjs");
|
|
5
|
+
const operators_1 = require("rxjs/operators");
|
|
6
|
+
/**
|
|
7
|
+
* Adaptive rate limiter for DuckDuckGo searches
|
|
8
|
+
* - Starts with immediate execution (no queue)
|
|
9
|
+
* - Activates queue on rate limit errors (202, 403, 429)
|
|
10
|
+
* - Automatically deactivates queue after success period
|
|
11
|
+
*/
|
|
12
|
+
class DuckDuckGoRateLimiter {
|
|
13
|
+
requestQueue$ = new rxjs_1.Subject();
|
|
14
|
+
isQueueActive = false;
|
|
15
|
+
lastRateLimitTime = 0;
|
|
16
|
+
resetQueueTimer;
|
|
17
|
+
activeRequests = 0;
|
|
18
|
+
config = {
|
|
19
|
+
maxConcurrent: 1, // When queue is active, serialize requests
|
|
20
|
+
queueTimeout: 30000, // 30 seconds timeout for queued requests
|
|
21
|
+
retryDelay: 2000, // 2 seconds between retries
|
|
22
|
+
maxRetries: 4,
|
|
23
|
+
resetQueueAfterMs: 10000 // Reset queue after 10 seconds of success
|
|
24
|
+
};
|
|
25
|
+
rateLimitStatusCodes = [202, 403, 429];
|
|
26
|
+
constructor(config) {
|
|
27
|
+
this.config = { ...this.config, ...config };
|
|
28
|
+
this.setupQueueProcessor();
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Process requests through the queue when active
|
|
32
|
+
*/
|
|
33
|
+
setupQueueProcessor() {
|
|
34
|
+
this.requestQueue$.pipe((0, operators_1.mergeMap)(({ request, resolver, rejecter }) => {
|
|
35
|
+
// If queue is not active and request allows immediate execution, process immediately
|
|
36
|
+
if (!this.isQueueActive && !request.disableQueueing) {
|
|
37
|
+
return this.executeRequest(request).pipe((0, operators_1.tap)({
|
|
38
|
+
next: (result) => resolver(result),
|
|
39
|
+
error: (error) => rejecter(error)
|
|
40
|
+
}), (0, operators_1.catchError)(() => rxjs_1.EMPTY) // Prevent stream termination
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
// Otherwise process through queue
|
|
44
|
+
return (0, rxjs_1.from)([{ request, resolver, rejecter }]).pipe((0, operators_1.concatMap)(({ request, resolver, rejecter }) => this.executeRequest(request).pipe((0, operators_1.tap)({
|
|
45
|
+
next: (result) => resolver(result),
|
|
46
|
+
error: (error) => rejecter(error)
|
|
47
|
+
}), (0, operators_1.catchError)(() => rxjs_1.EMPTY) // Prevent stream termination
|
|
48
|
+
)));
|
|
49
|
+
}, this.config.maxConcurrent)).subscribe();
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Execute a search request with retry logic
|
|
53
|
+
*/
|
|
54
|
+
executeRequest(request) {
|
|
55
|
+
return (0, rxjs_1.from)(fetch(request.url, request.options)).pipe((0, operators_1.timeout)(this.config.queueTimeout), (0, operators_1.mergeMap)(async (response) => {
|
|
56
|
+
const text = await response.text();
|
|
57
|
+
// Check for rate limit status codes
|
|
58
|
+
if (this.rateLimitStatusCodes.includes(response.status)) {
|
|
59
|
+
this.activateQueue();
|
|
60
|
+
throw new Error(`Rate limited: ${response.status}`);
|
|
61
|
+
}
|
|
62
|
+
// Success - consider deactivating queue
|
|
63
|
+
if (response.ok) {
|
|
64
|
+
this.scheduleQueueReset();
|
|
65
|
+
}
|
|
66
|
+
return { response, text };
|
|
67
|
+
}), (0, operators_1.retry)({
|
|
68
|
+
count: this.config.maxRetries,
|
|
69
|
+
delay: (error, retryCount) => {
|
|
70
|
+
// Exponential backoff for rate limit errors
|
|
71
|
+
const delayMs = this.isQueueActive
|
|
72
|
+
? this.config.retryDelay * Math.pow(2, retryCount - 1)
|
|
73
|
+
: 0;
|
|
74
|
+
return (0, rxjs_1.timer)(delayMs);
|
|
75
|
+
}
|
|
76
|
+
}), (0, operators_1.catchError)((error) => {
|
|
77
|
+
// If it's a rate limit error and queueing is disabled, fail immediately
|
|
78
|
+
if (request.disableQueueing && this.isQueueActive) {
|
|
79
|
+
return (0, rxjs_1.throwError)(() => new Error('Rate limited and queueing is disabled'));
|
|
80
|
+
}
|
|
81
|
+
return (0, rxjs_1.throwError)(() => error);
|
|
82
|
+
}));
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Activate the request queue due to rate limiting
|
|
86
|
+
*/
|
|
87
|
+
activateQueue() {
|
|
88
|
+
if (!this.isQueueActive) {
|
|
89
|
+
console.log('[DuckDuckGoRateLimiter] Activating queue due to rate limit');
|
|
90
|
+
this.isQueueActive = true;
|
|
91
|
+
this.lastRateLimitTime = Date.now();
|
|
92
|
+
}
|
|
93
|
+
// Cancel any pending queue reset
|
|
94
|
+
if (this.resetQueueTimer) {
|
|
95
|
+
clearTimeout(this.resetQueueTimer);
|
|
96
|
+
this.resetQueueTimer = undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Schedule queue deactivation after successful requests
|
|
101
|
+
*/
|
|
102
|
+
scheduleQueueReset() {
|
|
103
|
+
if (!this.isQueueActive)
|
|
104
|
+
return;
|
|
105
|
+
// Clear existing timer
|
|
106
|
+
if (this.resetQueueTimer) {
|
|
107
|
+
clearTimeout(this.resetQueueTimer);
|
|
108
|
+
}
|
|
109
|
+
// Schedule queue reset
|
|
110
|
+
this.resetQueueTimer = setTimeout(() => {
|
|
111
|
+
if (Date.now() - this.lastRateLimitTime >= this.config.resetQueueAfterMs) {
|
|
112
|
+
console.log('[DuckDuckGoRateLimiter] Deactivating queue - no rate limits detected');
|
|
113
|
+
this.isQueueActive = false;
|
|
114
|
+
this.resetQueueTimer = undefined;
|
|
115
|
+
}
|
|
116
|
+
}, this.config.resetQueueAfterMs);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Execute a search request through the rate limiter
|
|
120
|
+
*/
|
|
121
|
+
async search(request) {
|
|
122
|
+
return new Promise((resolve, reject) => {
|
|
123
|
+
// If queueing is disabled and queue is active, fail immediately
|
|
124
|
+
if (request.disableQueueing && this.isQueueActive) {
|
|
125
|
+
reject(new Error('Rate limiting active and queueing is disabled'));
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
// Process through the queue
|
|
129
|
+
this.requestQueue$.next({
|
|
130
|
+
request,
|
|
131
|
+
resolver: resolve,
|
|
132
|
+
rejecter: reject
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Get current queue status
|
|
138
|
+
*/
|
|
139
|
+
getStatus() {
|
|
140
|
+
return {
|
|
141
|
+
isQueueActive: this.isQueueActive,
|
|
142
|
+
activeRequests: this.activeRequests
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Force queue activation (for testing)
|
|
147
|
+
*/
|
|
148
|
+
forceActivateQueue() {
|
|
149
|
+
this.activateQueue();
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Force queue deactivation (for testing)
|
|
153
|
+
*/
|
|
154
|
+
forceDeactivateQueue() {
|
|
155
|
+
this.isQueueActive = false;
|
|
156
|
+
if (this.resetQueueTimer) {
|
|
157
|
+
clearTimeout(this.resetQueueTimer);
|
|
158
|
+
this.resetQueueTimer = undefined;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
exports.DuckDuckGoRateLimiter = DuckDuckGoRateLimiter;
|
|
163
|
+
// Singleton instance
|
|
164
|
+
let rateLimiterInstance = null;
|
|
165
|
+
/**
|
|
166
|
+
* Get the singleton rate limiter instance
|
|
167
|
+
*/
|
|
168
|
+
function getDuckDuckGoRateLimiter(config) {
|
|
169
|
+
if (!rateLimiterInstance) {
|
|
170
|
+
rateLimiterInstance = new DuckDuckGoRateLimiter(config);
|
|
171
|
+
}
|
|
172
|
+
return rateLimiterInstance;
|
|
173
|
+
}
|
|
174
|
+
exports.getDuckDuckGoRateLimiter = getDuckDuckGoRateLimiter;
|
|
175
|
+
//# sourceMappingURL=duckduckgo-rate-limiter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckduckgo-rate-limiter.js","sourceRoot":"","sources":["../../../src/custom/web/duckduckgo-rate-limiter.ts"],"names":[],"mappings":";;;AAAA,+BAA+E;AAC/E,8CAA6F;AAqB7F;;;;;GAKG;AACH,MAAa,qBAAqB;IACtB,aAAa,GAAG,IAAI,cAAO,EAI/B,CAAC;IAEG,aAAa,GAAG,KAAK,CAAC;IACtB,iBAAiB,GAAG,CAAC,CAAC;IACtB,eAAe,CAAkB;IACjC,cAAc,GAAG,CAAC,CAAC;IAEV,MAAM,GAAoB;QACvC,aAAa,EAAE,CAAC,EAAE,2CAA2C;QAC7D,YAAY,EAAE,KAAK,EAAE,yCAAyC;QAC9D,UAAU,EAAE,IAAI,EAAE,4BAA4B;QAC9C,UAAU,EAAE,CAAC;QACb,iBAAiB,EAAE,KAAK,CAAC,0CAA0C;KACtE,CAAC;IAEe,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAExD,YAAY,MAAiC;QACzC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QAC5C,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,mBAAmB;QACvB,IAAI,CAAC,aAAa,CAAC,IAAI,CACnB,IAAA,oBAAQ,EAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;YACzC,qFAAqF;YACrF,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAClD,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,IAAI,CACpC,IAAA,eAAG,EAAC;oBACA,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAClC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;iBACpC,CAAC,EACF,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,YAAK,CAAC,CAAC,6BAA6B;iBACxD,CAAC;YACN,CAAC;YAED,kCAAkC;YAClC,OAAO,IAAA,WAAI,EAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAC/C,IAAA,qBAAS,EAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC1C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,IAAI,CAC7B,IAAA,eAAG,EAAC;gBACA,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAClC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;aACpC,CAAC,EACF,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,YAAK,CAAC,CAAC,6BAA6B;aACxD,CACJ,CACJ,CAAC;QACN,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAChC,CAAC,SAAS,EAAE,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAsB;QACzC,OAAO,IAAA,WAAI,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACjD,IAAA,mBAAO,EAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EACjC,IAAA,oBAAQ,EAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YACxB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,oCAAoC;YACpC,IAAI,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACxD,CAAC;YAED,wCAAwC;YACxC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC9B,CAAC,CAAC,EACF,IAAA,iBAAK,EAAC;YACF,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC7B,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;gBACzB,4CAA4C;gBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa;oBAC9B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC;oBACtD,CAAC,CAAC,CAAC,CAAC;gBACR,OAAO,IAAA,YAAK,EAAC,OAAO,CAAC,CAAC;YAC1B,CAAC;SACJ,CAAC,EACF,IAAA,sBAAU,EAAC,CAAC,KAAK,EAAE,EAAE;YACjB,wEAAwE;YACxE,IAAI,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBAChD,OAAO,IAAA,iBAAU,EAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;YAChF,CAAC;YACD,OAAO,IAAA,iBAAU,EAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED;;OAEG;IACK,aAAa;QACjB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;YAC1E,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,CAAC;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACrC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,kBAAkB;QACtB,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAEhC,uBAAuB;QACvB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACvC,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,GAAG,EAAE;YACnC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBACvE,OAAO,CAAC,GAAG,CAAC,sEAAsE,CAAC,CAAC;gBACpF,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;gBAC3B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACrC,CAAC;QACL,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB;QAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,gEAAgE;YAChE,IAAI,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBAChD,MAAM,CAAC,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC,CAAC;gBACnE,OAAO;YACX,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;gBACpB,OAAO;gBACP,QAAQ,EAAE,OAAO;gBACjB,QAAQ,EAAE,MAAM;aACnB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO;YACH,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,kBAAkB;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,oBAAoB;QAChB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACrC,CAAC;IACL,CAAC;CACJ;AA3LD,sDA2LC;AAED,qBAAqB;AACrB,IAAI,mBAAmB,GAAiC,IAAI,CAAC;AAE7D;;GAEG;AACH,SAAgB,wBAAwB,CAAC,MAAiC;IACtE,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACvB,mBAAmB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC/B,CAAC;AALD,4DAKC"}
|
|
@@ -37,6 +37,7 @@ export declare class WebSearchAction extends BaseAction {
|
|
|
37
37
|
* - MaxResults: Maximum number of results to return (default: 10, max: 30)
|
|
38
38
|
* - Region: Regional search preference (default: 'us-en')
|
|
39
39
|
* - SafeSearch: Safe search level - 'strict', 'moderate', 'off' (default: 'moderate')
|
|
40
|
+
* - DisableQueueing: If true, fails immediately when rate limited instead of queueing (default: false)
|
|
40
41
|
*
|
|
41
42
|
* @returns Web search results with titles, URLs, and snippets
|
|
42
43
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-search.action.d.ts","sourceRoot":"","sources":["../../../src/custom/web/web-search.action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"web-search.action.d.ts","sourceRoot":"","sources":["../../../src/custom/web/web-search.action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAIrD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBACa,eAAgB,SAAQ,UAAU;IAE3C;;;;;;;;;;;OAWG;cACa,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA4KvF;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAkE3B;;OAEG;IACH,OAAO,CAAC,aAAa;IAQrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAiG1B;;OAEG;IACH,OAAO,CAAC,SAAS;IAcjB;;OAEG;IACH,OAAO,CAAC,SAAS;CAYpB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,SAElC"}
|