@smythos/sre 1.7.18 → 1.7.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/index.js +120 -82
  2. package/dist/index.js.map +1 -1
  3. package/dist/types/Components/DataSourceIndexer.class.d.ts +4 -12
  4. package/dist/types/Components/GenAILLM.class.d.ts +5 -5
  5. package/dist/types/Components/RAG/DataSourceCleaner.class.d.ts +37 -0
  6. package/dist/types/Components/RAG/DataSourceComponent.class.d.ts +30 -0
  7. package/dist/types/Components/RAG/DataSourceIndexer.class.d.ts +14 -0
  8. package/dist/types/Components/RAG/DataSourceLookup.class.d.ts +36 -0
  9. package/dist/types/Components/index.d.ts +3 -3
  10. package/dist/types/helpers/Conversation.helper.d.ts +3 -0
  11. package/dist/types/index.d.ts +3 -3
  12. package/dist/types/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.d.ts +1 -0
  13. package/dist/types/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.d.ts +11 -4
  14. package/dist/types/subsystems/IO/VectorDB.service/embed/index.d.ts +5 -0
  15. package/dist/types/subsystems/LLMManager/LLM.inference.d.ts +10 -3
  16. package/dist/types/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.d.ts +4 -2
  17. package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.d.ts +35 -0
  18. package/dist/types/subsystems/Security/Account.service/AccountConnector.d.ts +2 -2
  19. package/dist/types/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.d.ts +10 -0
  20. package/dist/types/subsystems/Security/Vault.service/connectors/SecretsManager.class.d.ts +6 -2
  21. package/dist/types/types/LLM.types.d.ts +2 -0
  22. package/dist/types/types/VectorDB.types.d.ts +4 -0
  23. package/dist/types/utils/array.utils.d.ts +4 -0
  24. package/dist/types/utils/string.utils.d.ts +1 -0
  25. package/package.json +3 -3
  26. package/src/Components/APIEndpoint.class.ts +1 -6
  27. package/src/Components/Component.class.ts +14 -1
  28. package/src/Components/DataSourceIndexer.class.ts +148 -34
  29. package/src/Components/GenAILLM.class.ts +21 -11
  30. package/src/Components/RAG/DataSourceCleaner.class.ts +178 -0
  31. package/src/Components/RAG/DataSourceComponent.class.ts +111 -0
  32. package/src/Components/RAG/DataSourceIndexer.class.ts +254 -0
  33. package/src/Components/{DataSourceLookup.class.ts → RAG/DataSourceLookup.class.ts} +92 -3
  34. package/src/Components/ServerlessCode.class.ts +1 -4
  35. package/src/Components/index.ts +3 -3
  36. package/src/helpers/AWSLambdaCode.helper.ts +40 -45
  37. package/src/helpers/Conversation.helper.ts +14 -10
  38. package/src/helpers/S3Cache.helper.ts +2 -1
  39. package/src/index.ts +212 -212
  40. package/src/index.ts.bak +212 -212
  41. package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +3 -1
  42. package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +145 -19
  43. package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +56 -22
  44. package/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts +1 -0
  45. package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +2 -1
  46. package/src/subsystems/IO/VectorDB.service/embed/index.ts +18 -0
  47. package/src/subsystems/LLMManager/LLM.inference.ts +63 -47
  48. package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +35 -10
  49. package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +12 -4
  50. package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +4 -4
  51. package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +105 -23
  52. package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +17 -5
  53. package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts +18 -3
  54. package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +14 -5
  55. package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +6 -4
  56. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts +5 -5
  57. package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +8 -3
  58. package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +9 -8
  59. package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts +126 -28
  60. package/src/subsystems/ObservabilityManager/Telemetry.service/connectors/OTel/OTel.class.ts +38 -6
  61. package/src/subsystems/Security/Account.service/AccountConnector.ts +3 -3
  62. package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +111 -48
  63. package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +41 -66
  64. package/src/types/LLM.types.ts +5 -0
  65. package/src/types/VectorDB.types.ts +4 -0
  66. package/src/utils/array.utils.ts +11 -0
  67. package/src/utils/base64.utils.ts +1 -1
  68. package/src/utils/string.utils.ts +3 -192
  69. package/src/Components/DataSourceCleaner.class.ts +0 -92
@@ -1,6 +1,5 @@
1
1
  import { ConnectorService } from '@sre/Core/ConnectorsService';
2
2
  import { Logger } from '@sre/helpers/Log.helper';
3
- //import { SmythRuntime } from '@sre/Core/SmythRuntime.class';
4
3
  import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
5
4
  import { ACL } from '@sre/Security/AccessControl/ACL.class';
6
5
  import { SecureConnector } from '@sre/Security/SecureConnector.class';
@@ -9,16 +8,19 @@ import { IAccessCandidate, TAccessLevel, TAccessRole } from '@sre/types/ACL.type
9
8
  import {
10
9
  CreateSecretCommand,
11
10
  DeleteSecretCommand,
11
+ DescribeSecretCommand,
12
+ DescribeSecretCommandOutput,
12
13
  GetSecretValueCommand,
13
14
  GetSecretValueCommandOutput,
14
15
  ListSecretsCommand,
15
16
  ListSecretsCommandOutput,
16
17
  PutSecretValueCommand,
18
+ RestoreSecretCommand,
17
19
  SecretsManagerClient,
18
20
  } from '@aws-sdk/client-secrets-manager';
19
- import { randomUUID } from 'crypto';
20
21
  import { ManagedVaultConnector } from '../ManagedVaultConnector';
21
22
  import { SecretsManagerConfig } from '../../Vault.service/connectors/SecretsManager.class';
23
+ const DELETION_MARKER_ERROR_MESSAGE = "You can't create this secret because a secret with this name is already scheduled for deletion."
22
24
 
23
25
  const console = Logger('SecretManagerManagedVault');
24
26
 
@@ -26,49 +28,87 @@ export class SecretManagerManagedVault extends ManagedVaultConnector {
26
28
  public name: string = 'SecretManagerManagedVault';
27
29
  public scope: string = 'smyth-managed-vault';
28
30
  private secretsManager: SecretsManagerClient;
31
+ private prefix: string;
29
32
 
30
- constructor(protected _settings: SecretsManagerConfig & { vaultName: string }) {
33
+ constructor(protected _settings: SecretsManagerConfig & { vaultName: string, prefix: string }) {
31
34
  super(_settings);
32
- //if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
33
35
 
36
+ this.prefix = _settings.prefix || '';
34
37
  this.secretsManager = new SecretsManagerClient({
35
38
  region: _settings.region,
36
39
  ...(_settings.awsAccessKeyId && _settings.awsSecretAccessKey
37
40
  ? {
38
- accessKeyId: _settings.awsAccessKeyId,
39
- secretAccessKey: _settings.awsSecretAccessKey,
40
- }
41
+ accessKeyId: _settings.awsAccessKeyId,
42
+ secretAccessKey: _settings.awsSecretAccessKey,
43
+ }
41
44
  : {}),
42
45
  });
43
46
  }
44
47
 
45
48
  @SecureConnector.AccessControl
46
49
  protected async get(acRequest: AccessRequest, secretName: string) {
47
- const secret = await this.getSecretByName(secretName);
48
- return secret?.SecretString;
50
+ try {
51
+ const secret = await this.getSecretByName(acRequest, secretName);
52
+ return secret?.value;
53
+ } catch (error) {
54
+ console.error(error);
55
+ return null;
56
+ }
49
57
  }
50
58
 
51
59
  @SecureConnector.AccessControl
52
60
  protected async set(acRequest: AccessRequest, secretName: string, value: string) {
53
- const secret = await this.getSecretByName(secretName);
61
+ const accountConnector = ConnectorService.getAccountConnector();
62
+ const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
63
+ const secret = await this.getSecretByName(acRequest, secretName);
54
64
  if (secret) {
55
- await this.secretsManager.send(new PutSecretValueCommand({ SecretId: secret.ARN, SecretString: value }));
65
+ await this.secretsManager.send(new PutSecretValueCommand({ SecretId: secret.secretId, SecretString: value }));
56
66
  } else {
57
- await this.secretsManager.send(
58
- new CreateSecretCommand({
59
- Name: `smyth/${randomUUID()}`,
60
- SecretString: JSON.stringify({ [secretName]: value }),
61
- Tags: [{ Key: this.scope, Value: 'true' }],
62
- })
63
- );
67
+ const vaultKey = this.getVaultKey(secretName, teamId);
68
+ try {
69
+ await this.secretsManager.send(
70
+ new CreateSecretCommand({
71
+ Name: vaultKey,
72
+ Description: `Smyth Managed Vault Secret`,
73
+ SecretString: JSON.stringify({ [secretName]: value }),
74
+ Tags: [{ Key: this.scope, Value: 'true' }],
75
+ })
76
+ );
77
+ } catch (error) {
78
+ console.warn(error.message);
79
+ if (error.message === DELETION_MARKER_ERROR_MESSAGE) {
80
+ // The secret already exists — check if it's pending deletion
81
+ const secretDesc: DescribeSecretCommandOutput = await this.secretsManager.send(new DescribeSecretCommand({ SecretId: vaultKey }));
82
+
83
+ if (secretDesc.DeletedDate) {
84
+ console.debug(`Secret ${vaultKey} is pending deletion, restoring...`);
85
+ await this.secretsManager.send(new RestoreSecretCommand({ SecretId: vaultKey }));
86
+ } else {
87
+ console.debug(`Secret ${vaultKey} already exists, updating value...`);
88
+ }
89
+
90
+ // Update or re-put the secret value
91
+ await this.secretsManager.send(
92
+ new PutSecretValueCommand({
93
+ SecretId: vaultKey,
94
+ SecretString: JSON.stringify({
95
+ [secretName]: value,
96
+ }),
97
+ })
98
+ );
99
+
100
+ } else {
101
+ throw error;
102
+ }
103
+ }
64
104
  }
65
105
  }
66
106
 
67
107
  @SecureConnector.AccessControl
68
108
  protected async delete(acRequest: AccessRequest, secretName: string) {
69
- const secret = await this.getSecretByName(secretName);
109
+ const secret = await this.getSecretByName(acRequest, secretName);
70
110
  if (secret) {
71
- await this.secretsManager.send(new DeleteSecretCommand({ SecretId: secret.ARN }));
111
+ await this.secretsManager.send(new DeleteSecretCommand({ SecretId: secret.secretId, RecoveryWindowInDays: 7 }));
72
112
  }
73
113
  }
74
114
 
@@ -91,13 +131,35 @@ export class SecretManagerManagedVault extends ManagedVaultConnector {
91
131
  return acl;
92
132
  }
93
133
 
94
- private async getSecretByName(secretName: string) {
134
+ private async getSecretByName(acRequest: AccessRequest, secretName: string) {
95
135
  try {
136
+
137
+ const accountConnector = ConnectorService.getAccountConnector();
138
+ const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
139
+ const vaultKey = this.getVaultKey(secretName, teamId);
140
+ let secret: GetSecretValueCommandOutput | null = null;
141
+ try {
142
+ const describeSecret: DescribeSecretCommandOutput = await this.secretsManager.send(new DescribeSecretCommand({ SecretId: vaultKey }));
143
+ if (describeSecret.DeletedDate) {
144
+ secret = null;
145
+ } else {
146
+ secret = await this.secretsManager.send(new GetSecretValueCommand({ SecretId: vaultKey }));
147
+ }
148
+ } catch (error) {
149
+ secret = null;
150
+ console.warn('Secret not found by Id, trying to get it by name');
151
+ }
152
+ if (secret) {
153
+ return this.getFormattedSecret(secret);
154
+ }
155
+
156
+ // if not found by Id, try to get it by name
96
157
  const secrets = [];
97
158
  let nextToken: string | undefined;
159
+ let listingVaultKey = this.getVaultKey('', teamId);
98
160
  do {
99
161
  const listResponse: ListSecretsCommandOutput = await this.secretsManager.send(
100
- new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'tag-key', Values: [this.scope] }] })
162
+ new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'tag-key', Values: [this.scope] }, { Key: 'name', Values: [listingVaultKey] }] })
101
163
  );
102
164
  if (listResponse.SecretList) {
103
165
  for (const secret of listResponse.SecretList) {
@@ -105,7 +167,6 @@ export class SecretManagerManagedVault extends ManagedVaultConnector {
105
167
  secrets.push({
106
168
  ARN: secret.ARN,
107
169
  Name: secret.Name,
108
- CreatedDate: secret.CreatedDate,
109
170
  });
110
171
  }
111
172
  }
@@ -113,42 +174,44 @@ export class SecretManagerManagedVault extends ManagedVaultConnector {
113
174
  nextToken = listResponse.NextToken;
114
175
  } while (nextToken);
115
176
 
116
- const formattedSecrets = [];
117
177
  const $promises = [];
118
178
  for (const secret of secrets) {
119
179
  $promises.push(getSpecificSecret(secret, this.secretsManager));
120
180
  }
121
181
  const results = await Promise.all($promises);
122
- for (const result of results) {
123
- formattedSecrets.push(result);
124
- }
125
- const secret = formattedSecrets.find((s) => s.Name === secretName);
126
- return secret;
182
+ const secretData = results.find((s) => s.key === secretName);
183
+ return secretData;
127
184
  } catch (error) {
128
185
  console.error(error);
129
186
  }
130
187
 
131
188
  async function getSpecificSecret(secret, secretsManager: SecretsManagerClient) {
132
189
  const data: GetSecretValueCommandOutput = await secretsManager.send(new GetSecretValueCommand({ SecretId: secret.ARN }));
133
- let secretString = data.SecretString;
134
- let secretName = secret.Name;
135
-
136
- if (secretString) {
137
- try {
138
- let parsedSecret = JSON.parse(secretString);
139
- if (Object.keys(parsedSecret).length === 1) {
140
- secretName = Object.keys(parsedSecret)[0];
141
- secretString = parsedSecret[secretName];
142
- }
143
- } catch (error) {}
144
- }
145
- return {
146
- Name: secretName,
147
- ARN: secret.ARN,
148
- CreatedDate: secret.CreatedDate,
149
- SecretId: secret.Name,
150
- SecretString: secretString,
151
- };
190
+ return this.getFormattedSecret(data);
191
+ }
192
+ }
193
+
194
+ getVaultKey(secretName: string, teamId: string) {
195
+ return `${this.prefix.length ? `${this.prefix}/` : ''}${teamId}/${secretName}`;
196
+ }
197
+
198
+ getFormattedSecret(secret: GetSecretValueCommandOutput) {
199
+ let secretString = secret.SecretString;
200
+ let secretName = secret.Name;
201
+
202
+ if (secretString) {
203
+ try {
204
+ let parsedSecret = JSON.parse(secretString);
205
+ if (Object.keys(parsedSecret).length === 1) {
206
+ secretName = Object.keys(parsedSecret)[0];
207
+ secretString = parsedSecret[secretName];
208
+ }
209
+ } catch (error) { }
152
210
  }
211
+ return {
212
+ secretId: secret.Name,
213
+ key: secretName,
214
+ value: secretString,
215
+ };
153
216
  }
154
217
  }
@@ -1,7 +1,5 @@
1
1
  import { ConnectorService } from '@sre/Core/ConnectorsService';
2
2
  import { Logger } from '@sre/helpers/Log.helper';
3
- //import { SmythRuntime } from '@sre/Core/SmythRuntime.class';
4
- import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
5
3
  import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
6
4
  import { ACL } from '@sre/Security/AccessControl/ACL.class';
7
5
  import { SecureConnector } from '@sre/Security/SecureConnector.class';
@@ -15,28 +13,30 @@ import {
15
13
  GetSecretValueCommandOutput,
16
14
  } from '@aws-sdk/client-secrets-manager';
17
15
 
16
+ const defaultPrefix = 'smythos';
18
17
  const console = Logger('SecretsManager');
19
18
 
20
19
  export type SecretsManagerConfig = {
21
20
  region: string;
22
21
  awsAccessKeyId?: string;
23
22
  awsSecretAccessKey?: string;
23
+ prefix?: string;
24
24
  };
25
25
  export class SecretsManager extends VaultConnector {
26
26
  public name: string = 'SecretsManager';
27
27
  private secretsManager: SecretsManagerClient;
28
+ private prefix: string;
28
29
 
29
30
  constructor(protected _settings: SecretsManagerConfig) {
30
31
  super(_settings);
31
- //if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
32
-
32
+ this.prefix = _settings.prefix || defaultPrefix;
33
33
  this.secretsManager = new SecretsManagerClient({
34
34
  region: _settings.region,
35
35
  ...(_settings.awsAccessKeyId && _settings.awsSecretAccessKey
36
36
  ? {
37
- accessKeyId: _settings.awsAccessKeyId,
38
- secretAccessKey: _settings.awsSecretAccessKey,
39
- }
37
+ accessKeyId: _settings.awsAccessKeyId,
38
+ secretAccessKey: _settings.awsSecretAccessKey,
39
+ }
40
40
  : {}),
41
41
  });
42
42
  }
@@ -44,8 +44,10 @@ export class SecretsManager extends VaultConnector {
44
44
  @SecureConnector.AccessControl
45
45
  protected async get(acRequest: AccessRequest, secretName: string) {
46
46
  try {
47
- const secret = await this.getSecretByName(secretName);
48
- return secret?.SecretString;
47
+ const accountConnector = ConnectorService.getAccountConnector();
48
+ const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
49
+ let secret = await this.getSecretById(teamId, secretName);
50
+ return secret;
49
51
  } catch (error) {
50
52
  console.error(error);
51
53
  throw error;
@@ -60,8 +62,25 @@ export class SecretsManager extends VaultConnector {
60
62
 
61
63
  @SecureConnector.AccessControl
62
64
  protected async listKeys(acRequest: AccessRequest) {
63
- console.warn('SecretsManager.listKeys is not implemented');
64
- return [];
65
+ const accountConnector = ConnectorService.getAccountConnector();
66
+ const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
67
+ const secrets = [];
68
+ let nextToken: string | undefined;
69
+
70
+ do {
71
+ const listResponse: ListSecretsCommandOutput = await this.secretsManager.send(
72
+ new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'name', Values: [this.getVaultKey(teamId, '')] }] })
73
+ );
74
+ if (listResponse.SecretList) {
75
+ for (const secret of listResponse.SecretList) {
76
+ if (secret.Name) {
77
+ secrets.push(this.extractSecretName(secret.Name, teamId, this.prefix));
78
+ }
79
+ }
80
+ }
81
+ nextToken = listResponse.NextToken;
82
+ } while (nextToken);
83
+ return secrets;
65
84
  }
66
85
 
67
86
  public async getResourceACL(resourceId: string, candidate: IAccessCandidate) {
@@ -77,64 +96,20 @@ export class SecretsManager extends VaultConnector {
77
96
  return acl;
78
97
  }
79
98
 
80
- private async getSecretByName(secretName: string) {
81
- try {
82
- const secrets = [];
83
- let nextToken: string | undefined;
84
- do {
85
- const listResponse: ListSecretsCommandOutput = await this.secretsManager.send(
86
- new ListSecretsCommand({ NextToken: nextToken, Filters: [{ Key: 'tag-key', Values: ['smyth-vault'] }] })
87
- );
88
- if (listResponse.SecretList) {
89
- for (const secret of listResponse.SecretList) {
90
- if (secret.Name) {
91
- secrets.push({
92
- ARN: secret.ARN,
93
- Name: secret.Name,
94
- CreatedDate: secret.CreatedDate,
95
- });
96
- }
97
- }
98
- }
99
- nextToken = listResponse.NextToken;
100
- } while (nextToken);
99
+ private getVaultKey(teamId: string, secretName: string) {
100
+ return `${this.prefix.length ? `${this.prefix}/` : ''}${teamId}/${secretName}`;
101
+ }
101
102
 
102
- const formattedSecrets = [];
103
- const $promises = [];
104
- for (const secret of secrets) {
105
- $promises.push(getSpecificSecret(secret, this.secretsManager));
106
- }
107
- const results = await Promise.all($promises);
108
- for (const result of results) {
109
- formattedSecrets.push(result);
110
- }
111
- const secret = formattedSecrets.find((s) => s.Name === secretName);
112
- return secret;
103
+ private async getSecretById(teamId: string, secretId: string) {
104
+ try {
105
+ const secret: GetSecretValueCommandOutput = await this.secretsManager.send(new GetSecretValueCommand({ SecretId: this.getVaultKey(teamId, secretId) }));
106
+ return secret.SecretString;
113
107
  } catch (error) {
114
- console.error(error);
108
+ return null;
115
109
  }
110
+ }
116
111
 
117
- async function getSpecificSecret(secret, secretsManager: SecretsManagerClient) {
118
- const data: GetSecretValueCommandOutput = await secretsManager.send(new GetSecretValueCommand({ SecretId: secret.ARN }));
119
- let secretString = data.SecretString;
120
- let secretName = secret.Name;
121
-
122
- if (secretString) {
123
- try {
124
- let parsedSecret = JSON.parse(secretString);
125
- if (Object.keys(parsedSecret).length === 1) {
126
- secretName = Object.keys(parsedSecret)[0];
127
- secretString = parsedSecret[secretName];
128
- }
129
- } catch (error) {}
130
- }
131
- return {
132
- Name: secretName,
133
- ARN: secret.ARN,
134
- CreatedDate: secret.CreatedDate,
135
- SecretId: secret.Name,
136
- SecretString: secretString,
137
- };
138
- }
112
+ private extractSecretName(secretKey: string, teamId: string, prefix: string) {
113
+ return secretKey.replace(`${prefix}/${teamId}/`, '');
139
114
  }
140
115
  }
@@ -80,6 +80,7 @@ export type TxAIToolsInfo = {
80
80
  };
81
81
  };
82
82
 
83
+ // #region TLLMParams
83
84
  export type TToolsInfo = {
84
85
  openai: TOpenAIToolsInfo;
85
86
  xai: TxAIToolsInfo;
@@ -189,6 +190,8 @@ type TLLMImageGenConfig = {
189
190
 
190
191
  export type TLLMParams = TLLMTextGenConfig & TLLMSearchConfig & TLLMImageGenConfig & TLLMMiscConfig & TLLMRuntimeContext;
191
192
 
193
+ // #endregion TLLMParams
194
+
192
195
  export type TLLMPreparedParams = TLLMParams & {
193
196
  body: any;
194
197
  modelEntryName?: string; // for usage reporting
@@ -344,6 +347,7 @@ export type ToolData = {
344
347
  function?: any;
345
348
  error?: string; // for Bedrock
346
349
  callId?: string; // for OpenAI Responses API call ID mapping
350
+ thoughtSignature?: string; // for Google AI - required to maintain reasoning context
347
351
  };
348
352
 
349
353
  /**
@@ -571,6 +575,7 @@ export interface TGoogleAIRequestBody {
571
575
  topK?: number;
572
576
  stopSequences?: string[];
573
577
  responseMimeType?: string;
578
+ media_resolution?: 'low' | 'medium' | 'high';
574
579
  };
575
580
  tools?: any;
576
581
  toolConfig?: any;
@@ -64,6 +64,10 @@ export interface IStorageVectorDataSource {
64
64
  id: string;
65
65
  candidateId: string;
66
66
  candidateRole: string;
67
+ datasourceSizeMb?: number;
68
+ createdAt?: Date;
69
+ chunkSize?: number;
70
+ chunkOverlap?: number;
67
71
  }
68
72
 
69
73
  export interface IStorageVectorNamespace {
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Utility method to chunk arrays into smaller batches
3
+ */
4
+ export function chunkArr<T>(arr: T[], sizePerChunk: number): T[][] {
5
+ return arr.reduce((chunks, elem, index) => {
6
+ const chunkIndex = Math.floor(index / sizePerChunk);
7
+ const chunk = chunks[chunkIndex] || [];
8
+ chunks[chunkIndex] = chunk.concat([elem]);
9
+ return chunks;
10
+ }, [] as T[][]);
11
+ }
@@ -110,7 +110,7 @@ export function isBase64(str: string): boolean {
110
110
  const cleanedBase64Data = cleanBase64(str);
111
111
 
112
112
  // Sometimes words like 'male' and hashes like md5, sha1, sha256, sha512 are detected as base64
113
- if (cleanedBase64Data.length < 128) return false;
113
+ //if (cleanedBase64Data.length < 128) return false;
114
114
 
115
115
  try {
116
116
  const buffer = Buffer.from(cleanedBase64Data, 'base64');
@@ -222,195 +222,6 @@ export const identifyMimetypeFromString = (input: string) => {
222
222
  return 'text/plain';
223
223
  };
224
224
 
225
- // export function chunkText(
226
- // text: string,
227
- // {
228
- // chunkSize,
229
- // chunkOverlap,
230
- // }: {
231
- // chunkSize: number;
232
- // chunkOverlap: number;
233
- // }
234
- // ): string[] {
235
- // const textSplitter = new TextSplitter({
236
- // chunkSize,
237
- // chunkOverlap,
238
- // separators: ['\n\n', '\n', ' ', ''],
239
- // keepSeparator: true,
240
- // });
241
- // let output = textSplitter.splitText(text);
242
-
243
- // return output;
244
- // }
245
- // class TextSplitter {
246
- // private chunkSize: number;
247
- // private chunkOverlap: number;
248
- // private separators: string[] = ['\n\n', '\n', ' ', ''];
249
- // private keepSeparator: boolean = true;
250
-
251
- // constructor({
252
- // chunkSize = 1000,
253
- // chunkOverlap = 200,
254
- // separators,
255
- // keepSeparator,
256
- // }: {
257
- // chunkSize?: number;
258
- // chunkOverlap?: number;
259
- // separators?: string[];
260
- // keepSeparator?: boolean;
261
- // } = {}) {
262
- // this.chunkSize = chunkSize;
263
- // this.chunkOverlap = chunkOverlap;
264
-
265
- // if (separators) {
266
- // this.separators = separators;
267
- // }
268
-
269
- // if (keepSeparator !== undefined) {
270
- // this.keepSeparator = keepSeparator;
271
- // }
272
-
273
- // if (this.chunkOverlap >= this.chunkSize) {
274
- // throw new Error('Cannot have chunkOverlap >= chunkSize');
275
- // }
276
- // }
277
-
278
- // public splitText(text: string): string[] {
279
- // return this._splitText(text, this.separators);
280
- // }
281
-
282
- // private _splitText(text: string, separators: string[]): string[] {
283
- // const finalChunks: string[] = [];
284
-
285
- // // Get appropriate separator to use
286
- // let separator: string = separators[separators.length - 1];
287
- // let newSeparators: string[] | undefined;
288
-
289
- // for (let i = 0; i < separators.length; i += 1) {
290
- // const s = separators[i];
291
- // if (s === '') {
292
- // separator = s;
293
- // break;
294
- // }
295
- // if (text.includes(s)) {
296
- // separator = s;
297
- // newSeparators = separators.slice(i + 1);
298
- // break;
299
- // }
300
- // }
301
-
302
- // // Split the text using the identified separator
303
- // const splits = this.splitOnSeparator(text, separator);
304
-
305
- // // Process splits, recursively splitting longer texts
306
- // let goodSplits: string[] = [];
307
- // const _separator = this.keepSeparator ? '' : separator;
308
-
309
- // for (const s of splits) {
310
- // if (this.lengthFunction(s) < this.chunkSize) {
311
- // goodSplits.push(s);
312
- // } else {
313
- // if (goodSplits.length) {
314
- // const mergedText = this.mergeSplits(goodSplits, _separator);
315
- // finalChunks.push(...mergedText);
316
- // goodSplits = [];
317
- // }
318
-
319
- // if (!newSeparators) {
320
- // finalChunks.push(s);
321
- // } else {
322
- // const otherInfo = this._splitText(s, newSeparators);
323
- // finalChunks.push(...otherInfo);
324
- // }
325
- // }
326
- // }
327
-
328
- // if (goodSplits.length) {
329
- // const mergedText = this.mergeSplits(goodSplits, _separator);
330
- // finalChunks.push(...mergedText);
331
- // }
332
-
333
- // return finalChunks;
334
- // }
335
-
336
- // private splitOnSeparator(text: string, separator: string): string[] {
337
- // let splits: string[];
338
-
339
- // if (separator) {
340
- // if (this.keepSeparator) {
341
- // const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
342
- // splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));
343
- // } else {
344
- // splits = text.split(separator);
345
- // }
346
- // } else {
347
- // splits = text.split('');
348
- // }
349
-
350
- // return splits.filter((s) => s !== '');
351
- // }
352
-
353
- // private lengthFunction(text: string): number {
354
- // return text.length;
355
- // }
356
-
357
- // private joinDocs(docs: string[], separator: string): string | null {
358
- // const text = docs.join(separator).trim();
359
- // return text === '' ? null : text;
360
- // }
361
-
362
- // private mergeSplits(splits: string[], separator: string): string[] {
363
- // const docs: string[] = [];
364
- // const currentDoc: string[] = [];
365
- // let total = 0;
366
-
367
- // for (const d of splits) {
368
- // const _len = this.lengthFunction(d);
369
-
370
- // if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
371
- // if (total > this.chunkSize) {
372
- // console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.chunkSize}`);
373
- // }
374
-
375
- // if (currentDoc.length > 0) {
376
- // const doc = this.joinDocs(currentDoc, separator);
377
- // if (doc !== null) {
378
- // docs.push(doc);
379
- // }
380
-
381
- // // Keep popping if conditions are met
382
- // while (total > this.chunkOverlap || (total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0)) {
383
- // total -= this.lengthFunction(currentDoc[0]);
384
- // currentDoc.shift();
385
- // }
386
- // }
387
- // }
388
-
389
- // currentDoc.push(d);
390
- // total += _len;
391
- // }
392
-
393
- // const doc = this.joinDocs(currentDoc, separator);
394
- // if (doc !== null) {
395
- // docs.push(doc);
396
- // }
397
-
398
- // return docs;
399
- // }
400
- // }
401
-
402
- // class RecursiveTextSplitter extends TextSplitter {
403
- // constructor({
404
- // chunkSize = 1000,
405
- // chunkOverlap = 200,
406
- // separators = ['\n\n', '\n', ' ', ''],
407
- // keepSeparator = true,
408
- // }: {
409
- // chunkSize?: number;
410
- // chunkOverlap?: number;
411
- // separators?: string[];
412
- // keepSeparator?: boolean;
413
- // } = {}) {
414
- // super({ chunkSize, chunkOverlap, separators, keepSeparator });
415
- // }
416
- // }
225
+ export function calcSizeMb(text: string): number {
226
+ return Buffer.byteLength(text, 'utf-8') / 1024 / 1024;
227
+ }