doers-comms-utils 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsBedrock.d.ts +9 -0
- package/dist/src/UtilsBedrock.js +89 -0
- package/dist/src/UtilsBee.d.ts +10 -0
- package/dist/src/UtilsBee.js +108 -0
- package/dist/src/UtilsDynamoDB.d.ts +34 -0
- package/dist/src/UtilsDynamoDB.js +381 -0
- package/dist/src/UtilsMisc.d.ts +34 -0
- package/dist/src/UtilsMisc.js +646 -0
- package/dist/src/UtilsS3.d.ts +13 -0
- package/dist/src/UtilsS3.js +96 -0
- package/dist/src/UtilsS3Vectors.d.ts +19 -0
- package/dist/src/UtilsS3Vectors.js +131 -0
- package/dist/src/index.d.ts +20 -0
- package/dist/src/index.js +26 -0
- package/dist/src/types/Bot.d.ts +28 -0
- package/dist/src/types/Bot.js +1 -0
- package/dist/src/types/BotDocument.d.ts +10 -0
- package/dist/src/types/BotDocument.js +1 -0
- package/dist/src/types/DoersCommsUtilsConfig.d.ts +8 -0
- package/dist/src/types/DoersCommsUtilsConfig.js +1 -0
- package/dist/src/types/Jurisdiction.d.ts +11 -0
- package/dist/src/types/Jurisdiction.js +1 -0
- package/dist/src/types/OfficeClass.d.ts +13 -0
- package/dist/src/types/OfficeClass.js +1 -0
- package/dist/test/Utils.test.d.ts +1 -0
- package/dist/test/Utils.test.js +170 -0
- package/eslint.config.ts +13 -0
- package/jest.config.js +20 -0
- package/package.json +43 -0
- package/src/UtilsBedrock.ts +93 -0
- package/src/UtilsBee.ts +115 -0
- package/src/UtilsDynamoDB.ts +450 -0
- package/src/UtilsMisc.ts +679 -0
- package/src/UtilsS3.ts +113 -0
- package/src/UtilsS3Vectors.ts +151 -0
- package/src/index.ts +32 -0
- package/src/types/Bot.ts +29 -0
- package/src/types/BotDocument.ts +10 -0
- package/src/types/DoersCommsUtilsConfig.ts +9 -0
- package/src/types/Jurisdiction.ts +11 -0
- package/src/types/OfficeClass.ts +14 -0
- package/test/Utils.test.ts +181 -0
- package/tsconfig.json +34 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
|
|
2
|
+
export declare class UtilsBedrock {
|
|
3
|
+
readonly bedrock: BedrockRuntime;
|
|
4
|
+
readonly text_decoder: TextDecoder;
|
|
5
|
+
constructor(region: string);
|
|
6
|
+
llama_invoke(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
|
|
7
|
+
gpt_converse(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
|
|
8
|
+
titan_invoke(text: string): Promise<string | undefined>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
|
|
2
|
+
export class UtilsBedrock {
|
|
3
|
+
bedrock;
|
|
4
|
+
text_decoder;
|
|
5
|
+
constructor(region) {
|
|
6
|
+
this.bedrock = new BedrockRuntime({ region: region });
|
|
7
|
+
this.text_decoder = new TextDecoder();
|
|
8
|
+
}
|
|
9
|
+
async llama_invoke(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
|
|
10
|
+
for (let i = 0; i < 3; i++) {
|
|
11
|
+
try {
|
|
12
|
+
const output = await this.bedrock.invokeModel({
|
|
13
|
+
modelId: model_id,
|
|
14
|
+
body: JSON.stringify({
|
|
15
|
+
prompt: prompt,
|
|
16
|
+
temperature: temperature,
|
|
17
|
+
top_p: top_p,
|
|
18
|
+
max_gen_len: max_gen_len
|
|
19
|
+
}),
|
|
20
|
+
contentType: "application/json"
|
|
21
|
+
});
|
|
22
|
+
const response = JSON.parse(this.text_decoder.decode(output.body));
|
|
23
|
+
return response.generation;
|
|
24
|
+
}
|
|
25
|
+
catch (e) {
|
|
26
|
+
console.log("Failed to get from Bedrock.");
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
console.log("Failed to get from Bedrock, quitting.");
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
async gpt_converse(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
|
|
33
|
+
for (let i = 0; i < 3; i++) {
|
|
34
|
+
try {
|
|
35
|
+
const response = await this.bedrock.converse({
|
|
36
|
+
modelId: model_id,
|
|
37
|
+
messages: [
|
|
38
|
+
{ role: "user", content: [{ text: prompt }] }
|
|
39
|
+
],
|
|
40
|
+
inferenceConfig: {
|
|
41
|
+
temperature: temperature,
|
|
42
|
+
topP: top_p,
|
|
43
|
+
maxTokens: max_gen_len
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
const output = response.output;
|
|
47
|
+
if (output === undefined || output.message === undefined || output.message.content === undefined) {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
for (const c of output.message.content) {
|
|
51
|
+
if (c.text !== undefined) {
|
|
52
|
+
return c.text;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error instanceof Error) {
|
|
58
|
+
console.log(error.stack);
|
|
59
|
+
}
|
|
60
|
+
console.log("Failed to get from Bedrock.");
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
console.log("Failed to get from Bedrock, quitting.");
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
async titan_invoke(text) {
|
|
67
|
+
for (let i = 0; i < 3; i++) {
|
|
68
|
+
try {
|
|
69
|
+
const output = await this.bedrock.invokeModel({
|
|
70
|
+
modelId: "amazon.titan-embed-text-v2:0",
|
|
71
|
+
body: JSON.stringify({
|
|
72
|
+
inputText: text
|
|
73
|
+
}),
|
|
74
|
+
contentType: "application/json"
|
|
75
|
+
});
|
|
76
|
+
const response = JSON.parse(this.text_decoder.decode(output.body));
|
|
77
|
+
return response.embedding;
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
if (error instanceof Error) {
|
|
81
|
+
console.log(error.stack);
|
|
82
|
+
}
|
|
83
|
+
console.log("Failed to get from Bedrock.");
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
console.log("Failed to get from Bedrock, quitting.");
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ScrapingBeeClient } from "scrapingbee";
|
|
2
|
+
export declare class UtilsBee {
|
|
3
|
+
readonly scraping_bee_api_key: string | undefined;
|
|
4
|
+
readonly bee: ScrapingBeeClient | undefined;
|
|
5
|
+
readonly text_decoder: TextDecoder;
|
|
6
|
+
constructor(scraping_bee_api_key: string | undefined);
|
|
7
|
+
get(url: string, params?: Record<string, string | number | boolean>): Promise<string | undefined>;
|
|
8
|
+
google_search(query: string, news?: boolean): Promise<Record<string, any>[] | undefined>;
|
|
9
|
+
youtube_search(query: string, options?: {}): Promise<Record<string, any>[] | undefined>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { ScrapingBeeClient } from "scrapingbee";
|
|
2
|
+
export class UtilsBee {
|
|
3
|
+
scraping_bee_api_key;
|
|
4
|
+
bee;
|
|
5
|
+
text_decoder;
|
|
6
|
+
constructor(scraping_bee_api_key) {
|
|
7
|
+
this.scraping_bee_api_key = scraping_bee_api_key;
|
|
8
|
+
this.bee = scraping_bee_api_key !== undefined ? new ScrapingBeeClient(scraping_bee_api_key) : undefined;
|
|
9
|
+
this.text_decoder = new TextDecoder();
|
|
10
|
+
}
|
|
11
|
+
async get(url, params = {}) {
|
|
12
|
+
if (this.bee === undefined) {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
for (let i = 0; i < 3; i++) {
|
|
16
|
+
try {
|
|
17
|
+
const response = await this.bee.htmlApi({
|
|
18
|
+
url: url,
|
|
19
|
+
params: params
|
|
20
|
+
});
|
|
21
|
+
if (response.status !== 200) {
|
|
22
|
+
console.log("Status", response.status);
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const text = this.text_decoder.decode(response.data);
|
|
26
|
+
return text;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error instanceof Error) {
|
|
30
|
+
console.log(error.stack);
|
|
31
|
+
}
|
|
32
|
+
console.log("Failed to get from Scraping Bee.");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
console.log("Failed to get from Scraping Bee, quitting.");
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
async google_search(query, news = false) {
|
|
39
|
+
if (this.bee === undefined) {
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
for (let i = 0; i < 3; i++) {
|
|
43
|
+
try {
|
|
44
|
+
const response = await this.bee.googleSearch({
|
|
45
|
+
search: query,
|
|
46
|
+
params: {
|
|
47
|
+
search_type: news ? "news" : undefined
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
if (response.status !== 200) {
|
|
51
|
+
console.log("Failed to Google:", response.status);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const data = JSON.parse(response.data.toString("utf-8"));
|
|
55
|
+
if (data.news_results === undefined || data.organic_results === undefined) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const results = news ? data.news_results : data.organic_results;
|
|
59
|
+
if (results === undefined) {
|
|
60
|
+
console.log("Failed to Google.");
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (!Array.isArray(results)) {
|
|
64
|
+
console.log("Results are not an array.");
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
return results;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
console.log("Failed to Google.");
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
console.log("Failed to Google thrice, quitting.");
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
async youtube_search(query, options = {}) {
|
|
77
|
+
if (this.bee === undefined) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
for (let i = 0; i < 3; i++) {
|
|
81
|
+
try {
|
|
82
|
+
const response = await this.bee.youtubeSearch({
|
|
83
|
+
search: query,
|
|
84
|
+
params: options
|
|
85
|
+
});
|
|
86
|
+
if (response.status !== 200) {
|
|
87
|
+
console.log("Failed to YouTube Search:", response.status);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const data = JSON.parse(response.data.toString("utf-8"));
|
|
91
|
+
const results = data.results;
|
|
92
|
+
if (results === undefined) {
|
|
93
|
+
console.log("Failed to YouTube Search.");
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (Array.isArray(results)) {
|
|
97
|
+
return results;
|
|
98
|
+
}
|
|
99
|
+
return JSON.parse(results);
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
console.log("Failed to YouTube Search.");
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
console.log("Failed to YouTube Search thrice, quitting.");
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { DynamoDB } from "@aws-sdk/client-dynamodb";
|
|
2
|
+
export declare class UtilsDynamoDB {
|
|
3
|
+
readonly dynamodb: DynamoDB;
|
|
4
|
+
constructor(region: string);
|
|
5
|
+
scan(table: string, options?: {
|
|
6
|
+
filters?: Record<string, any>;
|
|
7
|
+
undefined_attribute_names?: string[];
|
|
8
|
+
defined_attribute_names?: string[];
|
|
9
|
+
attribute_names?: string[];
|
|
10
|
+
concurrency?: number;
|
|
11
|
+
}): Promise<Record<string, any>[]>;
|
|
12
|
+
get(table: string, key: Record<string, any>, consistent?: boolean): Promise<Record<string, any> | undefined>;
|
|
13
|
+
get_max(table: string, primary_key: Record<string, any>): Promise<Record<string, any> | undefined>;
|
|
14
|
+
query(table: string, primary_key: Record<string, any>, options?: {
|
|
15
|
+
reverse?: boolean;
|
|
16
|
+
compile?: boolean;
|
|
17
|
+
}): Promise<Record<string, any>[]>;
|
|
18
|
+
query_prefix(table: string, primary_key: Record<string, any>, secondary_key_prefix: Record<string, string>, options?: {
|
|
19
|
+
reverse?: boolean;
|
|
20
|
+
compile?: boolean;
|
|
21
|
+
}): Promise<Record<string, any>[] | undefined>;
|
|
22
|
+
query_range(table: string, primary_key: Record<string, any>, secondary_key_range: Record<string, (string | number)[]>, options?: {
|
|
23
|
+
reverse?: boolean;
|
|
24
|
+
compile?: boolean;
|
|
25
|
+
}): Promise<Record<string, any>[]>;
|
|
26
|
+
set(table: string, key: Record<string, any>, attributes: Record<string, any>): Promise<void>;
|
|
27
|
+
append(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
|
|
28
|
+
add(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
|
|
29
|
+
remove(table: string, key: Record<string, any>, attributes: string[]): Promise<void>;
|
|
30
|
+
create(table: string, key: Record<string, any>, attributes?: Record<string, any[]>): Promise<void>;
|
|
31
|
+
delete(table: string, key: Record<string, any>): Promise<void>;
|
|
32
|
+
duplicate_attribute(table: string, attribute_name: string, new_attribute_name: string): Promise<void>;
|
|
33
|
+
remove_attribute(table: string, attribute_name: string): Promise<void>;
|
|
34
|
+
}
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
import { DynamoDB } from "@aws-sdk/client-dynamodb";
|
|
2
|
+
function convert_output(dynamodb_output) {
|
|
3
|
+
if (dynamodb_output.S !== undefined) {
|
|
4
|
+
return dynamodb_output.S;
|
|
5
|
+
}
|
|
6
|
+
else if (dynamodb_output.N !== undefined) {
|
|
7
|
+
return Number(dynamodb_output.N);
|
|
8
|
+
}
|
|
9
|
+
else if (dynamodb_output.L !== undefined) {
|
|
10
|
+
return dynamodb_output.L.map((a) => convert_output(a));
|
|
11
|
+
}
|
|
12
|
+
else if (dynamodb_output.SS !== undefined) {
|
|
13
|
+
return new Set(dynamodb_output.SS);
|
|
14
|
+
}
|
|
15
|
+
else if (dynamodb_output.M !== undefined) {
|
|
16
|
+
return Object.fromEntries(Object.entries(dynamodb_output.M).map(([key, value]) => [key, convert_output(value)]));
|
|
17
|
+
}
|
|
18
|
+
else if (dynamodb_output.BOOL !== undefined) {
|
|
19
|
+
return dynamodb_output.BOOL;
|
|
20
|
+
}
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
function not_undefined(x) {
|
|
24
|
+
return x !== undefined;
|
|
25
|
+
}
|
|
26
|
+
function is_item(x) {
|
|
27
|
+
return typeof x === "object" && Object.keys(x).filter(key => typeof key !== "string").length === 0 && !Array.isArray(x);
|
|
28
|
+
}
|
|
29
|
+
function convert_input(input) {
|
|
30
|
+
if (typeof input === "string") {
|
|
31
|
+
return { S: input };
|
|
32
|
+
}
|
|
33
|
+
else if (typeof input === "boolean") {
|
|
34
|
+
return { BOOL: input };
|
|
35
|
+
}
|
|
36
|
+
else if (typeof input === "number") {
|
|
37
|
+
return { N: input.toString() };
|
|
38
|
+
}
|
|
39
|
+
else if (Array.isArray(input)) {
|
|
40
|
+
const converted_list = input.map((a) => convert_input(a))
|
|
41
|
+
.filter(converted_input => converted_input !== undefined);
|
|
42
|
+
if (converted_list.length !== input.length) {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
return { L: converted_list };
|
|
46
|
+
}
|
|
47
|
+
else if (input instanceof Set) {
|
|
48
|
+
const converted_list = Array.from(input)
|
|
49
|
+
.filter(converted_input => typeof converted_input === "string");
|
|
50
|
+
if (converted_list.length !== input.size) {
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
return { SS: converted_list };
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
const converted_inputs = Object.fromEntries(Object.entries(input)
|
|
57
|
+
.filter(([key, value]) => value !== undefined && value !== null && key !== "")
|
|
58
|
+
.map(([key, value]) => [key, convert_input(value)])
|
|
59
|
+
.filter(([key, value]) => not_undefined(value)));
|
|
60
|
+
return {
|
|
61
|
+
M: converted_inputs
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
async function compile_pages(request, f, compile = true) {
|
|
66
|
+
const items = [];
|
|
67
|
+
let last_eval_key = undefined;
|
|
68
|
+
while (true) {
|
|
69
|
+
const request_page = {
|
|
70
|
+
...request,
|
|
71
|
+
ExclusiveStartKey: last_eval_key
|
|
72
|
+
};
|
|
73
|
+
const response = await f(request_page);
|
|
74
|
+
if (response.Items === undefined) {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
const new_items = response.Items.map(item => convert_output({ M: item }));
|
|
78
|
+
items.push(...new_items);
|
|
79
|
+
if (response.LastEvaluatedKey === undefined || !compile) {
|
|
80
|
+
return items;
|
|
81
|
+
}
|
|
82
|
+
last_eval_key = response.LastEvaluatedKey;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
export class UtilsDynamoDB {
|
|
86
|
+
dynamodb;
|
|
87
|
+
constructor(region) {
|
|
88
|
+
this.dynamodb = new DynamoDB({ region: region });
|
|
89
|
+
}
|
|
90
|
+
async scan(table, options = {}) {
|
|
91
|
+
const filters = options.filters !== undefined ? options.filters : {};
|
|
92
|
+
const undefined_attribute_names = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : [];
|
|
93
|
+
const defined_attribute_names = options.defined_attribute_names !== undefined ? options.defined_attribute_names : [];
|
|
94
|
+
const concurrency = options.concurrency !== undefined ? options.concurrency : 1;
|
|
95
|
+
const attribute_names = options.attribute_names !== undefined ? options.attribute_names : [];
|
|
96
|
+
const iterators = [];
|
|
97
|
+
for (let i = 0; i < concurrency; i++) {
|
|
98
|
+
const expression_attribute_names = Object.fromEntries([...Object.keys(filters), ...attribute_names, ...undefined_attribute_names, ...defined_attribute_names]
|
|
99
|
+
.map(attribute_name => ["#" + attribute_name, attribute_name]));
|
|
100
|
+
const expression_attribute_values = Object.fromEntries(Object.entries(filters)
|
|
101
|
+
.map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
|
|
102
|
+
.filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)));
|
|
103
|
+
const filter_expression = [
|
|
104
|
+
...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
|
|
105
|
+
...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
|
|
106
|
+
...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
|
|
107
|
+
].join(" AND ");
|
|
108
|
+
const projection_expression = attribute_names.map(attribute_name => "#" + attribute_name).join(", ");
|
|
109
|
+
const request = {
|
|
110
|
+
TableName: table,
|
|
111
|
+
ExpressionAttributeNames: Object.keys(expression_attribute_names).length > 0 ? expression_attribute_names : undefined,
|
|
112
|
+
ExpressionAttributeValues: Object.keys(expression_attribute_values).length > 0 ? expression_attribute_values : undefined,
|
|
113
|
+
FilterExpression: filter_expression.length > 0 ? filter_expression : undefined,
|
|
114
|
+
ProjectionExpression: projection_expression.length > 0 ? projection_expression : undefined,
|
|
115
|
+
Segment: i,
|
|
116
|
+
TotalSegments: concurrency
|
|
117
|
+
};
|
|
118
|
+
iterators.push(compile_pages(request, (request) => this.dynamodb.scan(request)));
|
|
119
|
+
}
|
|
120
|
+
const segments = await Promise.all(iterators);
|
|
121
|
+
const items = segments.flat().filter(is_item);
|
|
122
|
+
return items;
|
|
123
|
+
}
|
|
124
|
+
async get(table, key, consistent = false) {
|
|
125
|
+
const converted_key = convert_input(key)?.M;
|
|
126
|
+
if (converted_key === undefined) {
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
const item = await this.dynamodb.getItem({
|
|
130
|
+
ConsistentRead: consistent,
|
|
131
|
+
TableName: table,
|
|
132
|
+
Key: converted_key
|
|
133
|
+
})
|
|
134
|
+
.then(response => response.Item);
|
|
135
|
+
if (item === undefined) {
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
const converted_output = convert_output({ M: item });
|
|
139
|
+
if (!is_item(converted_output)) {
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
142
|
+
return converted_output;
|
|
143
|
+
}
|
|
144
|
+
async get_max(table, primary_key) {
|
|
145
|
+
if (Object.keys(primary_key).length !== 1) {
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
const key = Object.keys(primary_key)[0];
|
|
149
|
+
const value = convert_input(Object.values(primary_key)[0]);
|
|
150
|
+
if (value === undefined) {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
const request = {
|
|
154
|
+
TableName: table,
|
|
155
|
+
ExpressionAttributeNames: {
|
|
156
|
+
"#a": key
|
|
157
|
+
},
|
|
158
|
+
ExpressionAttributeValues: {
|
|
159
|
+
":a": value
|
|
160
|
+
},
|
|
161
|
+
KeyConditionExpression: "#a = :a",
|
|
162
|
+
Limit: 1,
|
|
163
|
+
ScanIndexForward: false
|
|
164
|
+
};
|
|
165
|
+
const items = await this.dynamodb.query(request)
|
|
166
|
+
.then(response => response.Items);
|
|
167
|
+
if (items === undefined || items[0] === undefined) {
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
const converted_output = convert_output({ M: items[0] });
|
|
171
|
+
if (!is_item(converted_output)) {
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
return converted_output;
|
|
175
|
+
}
|
|
176
|
+
async query(table, primary_key, options = {}) {
|
|
177
|
+
const reverse = options.reverse !== undefined ? options.reverse : false;
|
|
178
|
+
const compile = options.compile !== undefined ? options.compile : true;
|
|
179
|
+
if (Object.keys(primary_key).length !== 1) {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
182
|
+
const key = Object.keys(primary_key)[0];
|
|
183
|
+
const value = convert_input(Object.values(primary_key)[0]);
|
|
184
|
+
if (value === undefined) {
|
|
185
|
+
return [];
|
|
186
|
+
}
|
|
187
|
+
const request = {
|
|
188
|
+
TableName: table,
|
|
189
|
+
ExpressionAttributeNames: {
|
|
190
|
+
"#a": key
|
|
191
|
+
},
|
|
192
|
+
ExpressionAttributeValues: {
|
|
193
|
+
":a": value
|
|
194
|
+
},
|
|
195
|
+
KeyConditionExpression: "#a = :a",
|
|
196
|
+
ScanIndexForward: !reverse
|
|
197
|
+
};
|
|
198
|
+
return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
|
|
199
|
+
}
|
|
200
|
+
async query_prefix(table, primary_key, secondary_key_prefix, options = {}) {
|
|
201
|
+
const reverse = options.reverse !== undefined ? options.reverse : false;
|
|
202
|
+
const compile = options.compile !== undefined ? options.compile : true;
|
|
203
|
+
if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
const converted_primary_value = convert_input(Object.values(primary_key)[0]);
|
|
207
|
+
const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0]);
|
|
208
|
+
if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
const request = {
|
|
212
|
+
TableName: table,
|
|
213
|
+
ExpressionAttributeNames: {
|
|
214
|
+
"#a": Object.keys(primary_key)[0],
|
|
215
|
+
"#b": Object.keys(secondary_key_prefix)[0]
|
|
216
|
+
},
|
|
217
|
+
ExpressionAttributeValues: {
|
|
218
|
+
":a": converted_primary_value,
|
|
219
|
+
":b": converted_secondary_prefix_value
|
|
220
|
+
},
|
|
221
|
+
KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
|
|
222
|
+
ScanIndexForward: !reverse
|
|
223
|
+
};
|
|
224
|
+
return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
|
|
225
|
+
}
|
|
226
|
+
async query_range(table, primary_key, secondary_key_range, options = {}) {
|
|
227
|
+
const reverse = options.reverse !== undefined ? options.reverse : false;
|
|
228
|
+
const compile = options.compile !== undefined ? options.compile : true;
|
|
229
|
+
if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_range).length !== 1 || Object.values(secondary_key_range)[0].length !== 2) {
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
const converted_primary_value = convert_input(Object.values(primary_key)[0]);
|
|
233
|
+
const converted_secondary_range_start_value = convert_input(Object.values(secondary_key_range)[0][0]);
|
|
234
|
+
const converted_secondary_range_end_value = convert_input(Object.values(secondary_key_range)[0][1]);
|
|
235
|
+
if (converted_primary_value === undefined || converted_secondary_range_start_value === undefined || converted_secondary_range_end_value === undefined) {
|
|
236
|
+
return [];
|
|
237
|
+
}
|
|
238
|
+
const request = {
|
|
239
|
+
TableName: table,
|
|
240
|
+
ExpressionAttributeNames: {
|
|
241
|
+
"#a": Object.keys(primary_key)[0],
|
|
242
|
+
"#b": Object.keys(secondary_key_range)[0]
|
|
243
|
+
},
|
|
244
|
+
ExpressionAttributeValues: {
|
|
245
|
+
":a": converted_primary_value,
|
|
246
|
+
":b1": converted_secondary_range_start_value,
|
|
247
|
+
":b2": converted_secondary_range_end_value
|
|
248
|
+
},
|
|
249
|
+
KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
|
|
250
|
+
ScanIndexForward: !reverse
|
|
251
|
+
};
|
|
252
|
+
return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
|
|
253
|
+
}
|
|
254
|
+
async set(table, key, attributes) {
|
|
255
|
+
const converted_key = convert_input(key)?.M;
|
|
256
|
+
const request = {
|
|
257
|
+
TableName: table,
|
|
258
|
+
Key: converted_key,
|
|
259
|
+
UpdateExpression: "set " + Object.keys(attributes)
|
|
260
|
+
.filter(attribute_name => !Object.keys(key).includes(attribute_name))
|
|
261
|
+
.filter(attribute_name => attributes[attribute_name] !== undefined)
|
|
262
|
+
.map(attribute_name => "#" + attribute_name + " = :" + attribute_name).join(", "),
|
|
263
|
+
ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
|
|
264
|
+
.filter(attribute_name => !Object.keys(key).includes(attribute_name))
|
|
265
|
+
.filter(attribute_name => attributes[attribute_name] !== undefined)
|
|
266
|
+
.map(attribute_name => ["#" + attribute_name, attribute_name])),
|
|
267
|
+
ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
|
|
268
|
+
.filter(([attribute_name, attribute_value]) => !Object.keys(key).includes(attribute_name) && not_undefined(attribute_value))
|
|
269
|
+
.map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
|
|
270
|
+
.filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)))
|
|
271
|
+
};
|
|
272
|
+
await this.dynamodb.updateItem(request);
|
|
273
|
+
}
|
|
274
|
+
async append(table, key, attributes) {
|
|
275
|
+
const converted_key = convert_input(key)?.M;
|
|
276
|
+
const request = {
|
|
277
|
+
TableName: table,
|
|
278
|
+
Key: converted_key,
|
|
279
|
+
UpdateExpression: "set " + Object.keys(attributes)
|
|
280
|
+
.filter(attribute_name => attributes[attribute_name] !== undefined)
|
|
281
|
+
.map(attribute_name => "#" + attribute_name + " = list_append(#" + attribute_name + ", :" + attribute_name + ")").join(", "),
|
|
282
|
+
ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
|
|
283
|
+
.filter(attribute_name => attributes[attribute_name] !== undefined)
|
|
284
|
+
.map(attribute_name => ["#" + attribute_name, attribute_name])),
|
|
285
|
+
ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
|
|
286
|
+
.filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
|
|
287
|
+
.map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
|
|
288
|
+
.filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)))
|
|
289
|
+
};
|
|
290
|
+
this.dynamodb.updateItem(request);
|
|
291
|
+
}
|
|
292
|
+
async add(table, key, attributes) {
|
|
293
|
+
const item = await this.get(table, key, true);
|
|
294
|
+
if (item === undefined) {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const new_attributes = {};
|
|
298
|
+
for (const [attribute, values] of Object.entries(attributes)) {
|
|
299
|
+
if (item[attribute] === undefined) {
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
const new_values = values.filter(value => !item[attribute].includes(value));
|
|
303
|
+
if (new_values.length > 0) {
|
|
304
|
+
new_attributes[attribute] = new_values;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (Object.values(new_attributes).flat().length === 0) {
|
|
308
|
+
return undefined;
|
|
309
|
+
}
|
|
310
|
+
return await this.append(table, key, attributes);
|
|
311
|
+
}
|
|
312
|
+
async remove(table, key, attributes) {
|
|
313
|
+
const converted_key = convert_input(key)?.M;
|
|
314
|
+
const request = {
|
|
315
|
+
TableName: table,
|
|
316
|
+
Key: converted_key,
|
|
317
|
+
UpdateExpression: "remove " + attributes
|
|
318
|
+
.map(attribute_name => "#" + attribute_name).join(", "),
|
|
319
|
+
ExpressionAttributeNames: Object.fromEntries(attributes
|
|
320
|
+
.map(attribute_name => ["#" + attribute_name, attribute_name]))
|
|
321
|
+
};
|
|
322
|
+
await this.dynamodb.updateItem(request);
|
|
323
|
+
}
|
|
324
|
+
async create(table, key, attributes = {}) {
|
|
325
|
+
const item = await this.get(table, key, true);
|
|
326
|
+
if (item !== undefined) {
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
const converted_key = convert_input(key)?.M;
|
|
330
|
+
const converted_attributes = convert_input(attributes)?.M;
|
|
331
|
+
await this.dynamodb.putItem({
|
|
332
|
+
TableName: table,
|
|
333
|
+
Item: { ...converted_key, ...converted_attributes }
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
async delete(table, key) {
|
|
337
|
+
const converted_key = convert_input(key)?.M;
|
|
338
|
+
await this.dynamodb.deleteItem({
|
|
339
|
+
TableName: table,
|
|
340
|
+
Key: converted_key
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
async duplicate_attribute(table, attribute_name, new_attribute_name) {
|
|
344
|
+
const table_metadata = await this.dynamodb.describeTable({
|
|
345
|
+
TableName: table
|
|
346
|
+
});
|
|
347
|
+
if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
|
|
351
|
+
.filter(table_key_name => table_key_name !== undefined);
|
|
352
|
+
const items = await this.scan(table, { attribute_names: table_key_names.concat([attribute_name, new_attribute_name]) });
|
|
353
|
+
if (items.filter(item => item[new_attribute_name] !== undefined).length > 0) {
|
|
354
|
+
console.log("Cannot rename.", new_attribute_name, "is an existing item.");
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
for (const item of items) {
|
|
358
|
+
if (item[attribute_name] === undefined) {
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]));
|
|
362
|
+
await this.set(table, key, { [new_attribute_name]: item[attribute_name] });
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
async remove_attribute(table, attribute_name) {
|
|
366
|
+
const table_metadata = await this.dynamodb.describeTable({
|
|
367
|
+
TableName: table
|
|
368
|
+
});
|
|
369
|
+
if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
|
|
373
|
+
.filter(table_key_name => table_key_name !== undefined);
|
|
374
|
+
const items = await this.scan(table)
|
|
375
|
+
.then(items => items.filter(item => item[attribute_name] !== undefined));
|
|
376
|
+
for (const item of items) {
|
|
377
|
+
const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]));
|
|
378
|
+
await this.remove(table, key, [attribute_name]);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import * as nodemailer from "nodemailer";
|
|
2
|
+
import { DoersCommsUtilsConfig } from "./types/DoersCommsUtilsConfig.js";
|
|
3
|
+
interface Encryption {
|
|
4
|
+
iv: string;
|
|
5
|
+
ciphertext: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class UtilsMisc {
|
|
8
|
+
readonly config: DoersCommsUtilsConfig;
|
|
9
|
+
readonly text_encoder: TextEncoder;
|
|
10
|
+
readonly transporter: nodemailer.Transporter<import("nodemailer/lib/smtp-transport").SentMessageInfo, import("nodemailer/lib/smtp-transport").Options> | undefined;
|
|
11
|
+
constructor(config: DoersCommsUtilsConfig);
|
|
12
|
+
wait(duration: number): Promise<unknown>;
|
|
13
|
+
get_current_time(): string;
|
|
14
|
+
get_current_milliseconds(): number;
|
|
15
|
+
send_email(recipient: string, subject: string, text: string): Promise<boolean>;
|
|
16
|
+
admin_alert(text: string): Promise<boolean>;
|
|
17
|
+
safe_run(f: () => Promise<any>): Promise<void>;
|
|
18
|
+
iterate<T>(inputs: T[], f: (input: T) => Promise<any>, concurrency?: number, print_indices?: boolean): Promise<void>;
|
|
19
|
+
sha256(input: string): Promise<string>;
|
|
20
|
+
encrypt(text: string): Encryption;
|
|
21
|
+
decrypt(encryption: Encryption): string;
|
|
22
|
+
get_secret_hash(username: string): string;
|
|
23
|
+
get_election_id(year: string, office: string, state: string, district: string): string | undefined;
|
|
24
|
+
get_chunk_indices(text_length: number, max_length?: number, overlap?: number): [number, number][];
|
|
25
|
+
chunkify(text: string, max_length?: number, overlap?: number): {
|
|
26
|
+
chunk_index: [number, number];
|
|
27
|
+
chunk_text: string;
|
|
28
|
+
}[];
|
|
29
|
+
chunkify_lined(text: string, max_length?: number, overlap?: number): {
|
|
30
|
+
chunk_index: [number, number];
|
|
31
|
+
chunk_text: string;
|
|
32
|
+
}[];
|
|
33
|
+
}
|
|
34
|
+
export {};
|