@harperfast/harper-pro 5.0.5 → 5.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/bin/status.js +2 -2
- package/core/bin/stop.js +5 -6
- package/core/components/EntryHandler.ts +4 -2
- package/core/components/Scope.ts +1 -1
- package/core/components/componentLoader.ts +11 -4
- package/core/components/requestRestart.ts +17 -2
- package/core/dataLayer/harperBridge/TableSizeObject.ts +35 -0
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts +24 -0
- package/core/package-lock.json +1020 -101
- package/core/resources/DatabaseTransaction.ts +8 -3
- package/core/resources/Table.ts +12 -17
- package/core/resources/databases.ts +2 -2
- package/core/resources/graphql.ts +163 -165
- package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts +14 -3
- package/core/resources/indexes/vector.ts +17 -0
- package/core/resources/loadEnv.ts +20 -16
- package/core/resources/login.ts +4 -3
- package/core/resources/roles.ts +60 -65
- package/core/security/auth.ts +15 -14
- package/core/security/jsLoader.ts +27 -9
- package/core/server/REST.ts +10 -11
- package/core/server/fastifyRoutes.ts +30 -29
- package/core/server/graphqlQuerying.ts +4 -3
- package/core/server/http.ts +175 -1
- package/core/server/mqtt.ts +8 -2
- package/core/server/serverHelpers/serverUtilities.ts +2 -5
- package/core/server/threads/threadServer.js +30 -2
- package/core/server/throttle.ts +18 -0
- package/core/utility/environment/environmentManager.js +10 -4
- package/core/utility/environment/systemInformation.ts +698 -0
- package/core/utility/hdbTerms.ts +1 -0
- package/core/utility/operation_authorization.js +2 -5
- package/dist/core/bin/status.js +2 -2
- package/dist/core/bin/status.js.map +1 -1
- package/dist/core/bin/stop.js +5 -5
- package/dist/core/bin/stop.js.map +1 -1
- package/dist/core/components/EntryHandler.js +4 -2
- package/dist/core/components/EntryHandler.js.map +1 -1
- package/dist/core/components/Scope.js +1 -1
- package/dist/core/components/Scope.js.map +1 -1
- package/dist/core/components/componentLoader.js +11 -3
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/requestRestart.js +12 -1
- package/dist/core/components/requestRestart.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/TableSizeObject.js +32 -0
- package/dist/core/dataLayer/harperBridge/TableSizeObject.js.map +1 -0
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +18 -19
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js.map +1 -1
- package/dist/core/resources/DatabaseTransaction.js +6 -1
- package/dist/core/resources/DatabaseTransaction.js.map +1 -1
- package/dist/core/resources/Table.js +14 -18
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/databases.js +2 -1
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/resources/graphql.js +176 -176
- package/dist/core/resources/graphql.js.map +1 -1
- package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js +14 -2
- package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/core/resources/indexes/vector.js +14 -0
- package/dist/core/resources/indexes/vector.js.map +1 -1
- package/dist/core/resources/loadEnv.js +20 -17
- package/dist/core/resources/loadEnv.js.map +1 -1
- package/dist/core/resources/login.js +4 -4
- package/dist/core/resources/login.js.map +1 -1
- package/dist/core/resources/roles.js +64 -68
- package/dist/core/resources/roles.js.map +1 -1
- package/dist/core/security/auth.js +17 -15
- package/dist/core/security/auth.js.map +1 -1
- package/dist/core/security/jsLoader.js +29 -9
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/server/REST.js +11 -11
- package/dist/core/server/REST.js.map +1 -1
- package/dist/core/server/fastifyRoutes.js +30 -29
- package/dist/core/server/fastifyRoutes.js.map +1 -1
- package/dist/core/server/graphqlQuerying.js +5 -4
- package/dist/core/server/graphqlQuerying.js.map +1 -1
- package/dist/core/server/http.js +179 -0
- package/dist/core/server/http.js.map +1 -1
- package/dist/core/server/mqtt.js +5 -3
- package/dist/core/server/mqtt.js.map +1 -1
- package/dist/core/server/serverHelpers/serverUtilities.js +2 -2
- package/dist/core/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/core/server/threads/threadServer.js +26 -2
- package/dist/core/server/threads/threadServer.js.map +1 -1
- package/dist/core/server/throttle.js +17 -0
- package/dist/core/server/throttle.js.map +1 -1
- package/dist/core/utility/environment/environmentManager.js +9 -4
- package/dist/core/utility/environment/environmentManager.js.map +1 -1
- package/dist/core/utility/environment/systemInformation.js +359 -219
- package/dist/core/utility/environment/systemInformation.js.map +1 -1
- package/dist/core/utility/hdbTerms.js +1 -0
- package/dist/core/utility/hdbTerms.js.map +1 -1
- package/dist/core/utility/operation_authorization.js +2 -2
- package/dist/core/utility/operation_authorization.js.map +1 -1
- package/dist/licensing/usageLicensing.js +1 -1
- package/dist/licensing/usageLicensing.js.map +1 -1
- package/licensing/usageLicensing.ts +1 -1
- package/npm-shrinkwrap.json +982 -62
- package/package.json +2 -1
- package/studio/web/assets/{index-D07pIqJt.js → index-qbLPhOzw.js} +2 -2
- package/studio/web/assets/{index-D07pIqJt.js.map → index-qbLPhOzw.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +0 -25
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js +0 -34
- package/core/utility/environment/systemInformation.js +0 -355
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js +0 -24
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js.map +0 -1
|
@@ -256,9 +256,14 @@ export class DatabaseTransaction implements Transaction {
|
|
|
256
256
|
if (!outstandingCommit) {
|
|
257
257
|
outstandingCommit = commitResolution;
|
|
258
258
|
outstandingCommitStart = performance.now();
|
|
259
|
-
outstandingCommit
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
outstandingCommit
|
|
260
|
+
// if `commitResolution` rejects with and `ERR_BUSY` error, the retry logic
|
|
261
|
+
// will correct course, but the reject will still be propagated on the
|
|
262
|
+
// `outstandingCommit` promise and needs to be caught and silenced
|
|
263
|
+
.catch(() => {})
|
|
264
|
+
.finally(() => {
|
|
265
|
+
outstandingCommit = null;
|
|
266
|
+
});
|
|
262
267
|
}
|
|
263
268
|
const completions = [];
|
|
264
269
|
return commitResolution.then(
|
package/core/resources/Table.ts
CHANGED
|
@@ -91,6 +91,7 @@ const NULL_WITH_TIMESTAMP = new Uint8Array(9);
|
|
|
91
91
|
NULL_WITH_TIMESTAMP[8] = 0xc0; // null
|
|
92
92
|
const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
|
|
93
93
|
const RECORD_PRUNING_INTERVAL = 60000; // one minute
|
|
94
|
+
const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
|
|
94
95
|
envMngr.initSync();
|
|
95
96
|
const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
|
|
96
97
|
const LOCK_TIMEOUT = 10000;
|
|
@@ -4038,15 +4039,14 @@ export function makeTable(options) {
|
|
|
4038
4039
|
if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables');
|
|
4039
4040
|
if (updatedRecord.status > 0 && updatedRecord.headers) {
|
|
4040
4041
|
// if the source has a status code and headers, treat it as a response
|
|
4041
|
-
|
|
4042
|
-
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
} // there are definitely more status codes to handle
|
|
4042
|
+
const status = updatedRecord.status;
|
|
4043
|
+
if (status === 304) {
|
|
4044
|
+
// revalidation of our current cached record
|
|
4045
|
+
updatedRecord = existingRecord;
|
|
4046
|
+
version = existingVersion;
|
|
4047
|
+
} else if (!CACHEABLE_STATUS_CODES.has(status)) {
|
|
4048
|
+
// non-cacheable status - propagate to client without caching
|
|
4049
|
+
throw new ServerError(updatedRecord.body || 'Error from source', status);
|
|
4050
4050
|
} else {
|
|
4051
4051
|
let headers: any;
|
|
4052
4052
|
const sourceHeaders = updatedRecord.headers;
|
|
@@ -4074,16 +4074,11 @@ export function makeTable(options) {
|
|
|
4074
4074
|
if (data !== undefined) {
|
|
4075
4075
|
// we have structured data that we have parsed
|
|
4076
4076
|
delete headers['content-type']; // don't store the content type if we have already parsed it
|
|
4077
|
-
updatedRecord = {
|
|
4078
|
-
headers,
|
|
4079
|
-
data,
|
|
4080
|
-
};
|
|
4077
|
+
updatedRecord = { headers, data };
|
|
4081
4078
|
} else {
|
|
4082
|
-
updatedRecord = {
|
|
4083
|
-
headers,
|
|
4084
|
-
body: createBlob(updatedRecord.body),
|
|
4085
|
-
};
|
|
4079
|
+
updatedRecord = { headers, body: createBlob(updatedRecord.body) };
|
|
4086
4080
|
}
|
|
4081
|
+
if (status !== 200) updatedRecord.status = status;
|
|
4087
4082
|
}
|
|
4088
4083
|
}
|
|
4089
4084
|
if (typeof updatedRecord.toJSON === 'function') updatedRecord = updatedRecord.toJSON();
|
|
@@ -184,7 +184,6 @@ export function getDatabases(): Databases {
|
|
|
184
184
|
getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
|
|
185
185
|
(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
|
|
186
186
|
if (!databasePath) return;
|
|
187
|
-
|
|
188
187
|
if (existsSync(databasePath)) {
|
|
189
188
|
// First load all the databases from our main database folder
|
|
190
189
|
// TODO: Load any databases defined with explicit storage paths from the config
|
|
@@ -368,7 +367,7 @@ function readRocksMetaDb(path: string, defaultTable?: string, databaseName: stri
|
|
|
368
367
|
if (rootStore) {
|
|
369
368
|
initStores(path, rootStore, databaseName, defaultTable);
|
|
370
369
|
} else {
|
|
371
|
-
rootStore = openRocksDatabase(path, { disableWAL: false }) as RocksDatabaseEx;
|
|
370
|
+
rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as RocksDatabaseEx;
|
|
372
371
|
rocksdbDatabaseEnvs.set(path, rootStore);
|
|
373
372
|
initStores(path, rootStore, databaseName, defaultTable);
|
|
374
373
|
replayLogs(rootStore, databases[databaseName]);
|
|
@@ -723,6 +722,7 @@ export function database({ database: databaseName, table: tableName }) {
|
|
|
723
722
|
if (!rootStore || rootStore.status === 'closed') {
|
|
724
723
|
rootStore = openRocksDatabase(path, {
|
|
725
724
|
disableWAL: false,
|
|
725
|
+
enableStats: true,
|
|
726
726
|
});
|
|
727
727
|
rocksdbDatabaseEnvs.set(path, rootStore);
|
|
728
728
|
}
|
|
@@ -35,187 +35,185 @@ server.knownGraphQLDirectives.push(
|
|
|
35
35
|
* @param filePath
|
|
36
36
|
* @param resources
|
|
37
37
|
*/
|
|
38
|
-
export function
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
};
|
|
38
|
+
export function handleApplication(scope: import('../components/Scope.ts').Scope) {
|
|
39
|
+
scope.handleEntry(async (entry) => {
|
|
40
|
+
if (entry.eventType === 'unlink') return;
|
|
41
|
+
await processGraphQLSchema(entry.contents, entry.urlPath, entry.absolutePath, scope.resources);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
43
44
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}
|
|
67
|
-
if (typeDef.schema) typeDef.database = typeDef.schema;
|
|
68
|
-
if (!typeDef.table) typeDef.table = typeName;
|
|
69
|
-
if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
|
|
70
|
-
typeDef.attributes = typeDef.properties;
|
|
71
|
-
tables.push(typeDef);
|
|
72
|
-
}
|
|
73
|
-
if (directive.name.value === 'sealed') typeDef.sealed = true;
|
|
74
|
-
if (directive.name.value === 'splitSegments') typeDef.splitSegments = true;
|
|
75
|
-
if (directive.name.value === 'replicate') typeDef.replicate = true;
|
|
76
|
-
if (directive.name.value === 'export') {
|
|
77
|
-
typeDef.export = true;
|
|
78
|
-
for (const arg of directive.arguments) {
|
|
79
|
-
if (typeof typeDef.export !== 'object') typeDef.export = {};
|
|
80
|
-
typeDef.export[arg.name.value] = (arg.value as StringValueNode).value;
|
|
81
|
-
}
|
|
45
|
+
async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
|
|
46
|
+
// lazy load the graphql package so we don't load it for users that don't use graphql
|
|
47
|
+
const { parse, Source, Kind } = await import('graphql');
|
|
48
|
+
const ast = parse(new Source(gqlContent.toString(), filePath));
|
|
49
|
+
const types = new Map();
|
|
50
|
+
const tables = [];
|
|
51
|
+
// we begin by iterating through the definitions in the AST to get the types and convert them
|
|
52
|
+
// to a friendly format for table attributes
|
|
53
|
+
for (const definition of ast.definitions) {
|
|
54
|
+
switch (definition.kind) {
|
|
55
|
+
case Kind.OBJECT_TYPE_DEFINITION:
|
|
56
|
+
const typeName = definition.name.value;
|
|
57
|
+
// use type name as the default table
|
|
58
|
+
const properties = [];
|
|
59
|
+
const typeDef = { table: null, database: null, properties };
|
|
60
|
+
types.set(typeName, typeDef);
|
|
61
|
+
resources.allTypes.set(typeName, typeDef);
|
|
62
|
+
for (const directive of definition.directives) {
|
|
63
|
+
const directiveName = directive.name.value;
|
|
64
|
+
if (directiveName === 'table') {
|
|
65
|
+
for (const arg of directive.arguments) {
|
|
66
|
+
typeDef[arg.name.value] = (arg.value as StringValueNode).value;
|
|
82
67
|
}
|
|
68
|
+
if (typeDef.schema) typeDef.database = typeDef.schema;
|
|
69
|
+
if (!typeDef.table) typeDef.table = typeName;
|
|
70
|
+
if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
|
|
71
|
+
typeDef.attributes = typeDef.properties;
|
|
72
|
+
tables.push(typeDef);
|
|
83
73
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
return {
|
|
93
|
-
type: 'array',
|
|
94
|
-
elements: getProperty(type.type),
|
|
95
|
-
};
|
|
74
|
+
if (directive.name.value === 'sealed') typeDef.sealed = true;
|
|
75
|
+
if (directive.name.value === 'splitSegments') typeDef.splitSegments = true;
|
|
76
|
+
if (directive.name.value === 'replicate') typeDef.replicate = true;
|
|
77
|
+
if (directive.name.value === 'export') {
|
|
78
|
+
typeDef.export = true;
|
|
79
|
+
for (const arg of directive.arguments) {
|
|
80
|
+
if (typeof typeDef.export !== 'object') typeDef.export = {};
|
|
81
|
+
typeDef.export[arg.name.value] = (arg.value as StringValueNode).value;
|
|
96
82
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
let hasPrimaryKey = false;
|
|
86
|
+
function getProperty(type) {
|
|
87
|
+
if (type.kind === 'NonNullType') {
|
|
88
|
+
const property = getProperty(type.type);
|
|
89
|
+
property.nullable = false;
|
|
100
90
|
return property;
|
|
101
91
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
92
|
+
if (type.kind === 'ListType') {
|
|
93
|
+
return {
|
|
94
|
+
type: 'array',
|
|
95
|
+
elements: getProperty(type.type),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
const typeName = (type as NamedTypeNode).name?.value;
|
|
99
|
+
const property = { type: typeName };
|
|
100
|
+
Object.defineProperty(property, 'location', { value: type.loc.startToken });
|
|
101
|
+
return property;
|
|
102
|
+
}
|
|
103
|
+
const attributesObject = {};
|
|
104
|
+
for (const field of definition.fields) {
|
|
105
|
+
const property = getProperty(field.type);
|
|
106
|
+
property.name = field.name.value;
|
|
107
|
+
properties.push(property);
|
|
108
|
+
attributesObject[property.name] = undefined; // this is used as a backup scope for computed properties
|
|
109
|
+
for (const directive of field.directives) {
|
|
110
|
+
const directiveName = directive.name.value;
|
|
111
|
+
if (directiveName === 'primaryKey') {
|
|
112
|
+
if (hasPrimaryKey) console.warn('Can not define two attributes as a primary key at', directive.loc);
|
|
113
|
+
else {
|
|
114
|
+
property.isPrimaryKey = true;
|
|
115
|
+
hasPrimaryKey = true;
|
|
116
|
+
}
|
|
117
|
+
} else if (directiveName === 'indexed') {
|
|
118
|
+
const indexedDefinition = {};
|
|
119
|
+
// store indexed arguments for configurable indexes.
|
|
120
|
+
for (const arg of directive.arguments || []) {
|
|
121
|
+
indexedDefinition[arg.name.value] = (arg.value as StringValueNode).value;
|
|
122
|
+
}
|
|
123
|
+
property.indexed = indexedDefinition;
|
|
124
|
+
} else if (directiveName === 'computed') {
|
|
125
|
+
for (const arg of directive.arguments || []) {
|
|
126
|
+
if (arg.name.value === 'from') {
|
|
127
|
+
const computedFromExpression = (arg.value as StringValueNode).value;
|
|
128
|
+
property.computed = {
|
|
129
|
+
from: createComputedFrom(computedFromExpression, arg, attributesObject),
|
|
130
|
+
};
|
|
131
|
+
// if the version is not defined, we use the computed from expression as the version, any changes to the computed from expression will trigger a version change and reindex
|
|
132
|
+
if (property.version == undefined) property.version = computedFromExpression;
|
|
133
|
+
} else if (arg.name.value === 'version') {
|
|
134
|
+
property.version = (arg.value as StringValueNode).value;
|
|
141
135
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
136
|
+
}
|
|
137
|
+
property.computed = property.computed || true;
|
|
138
|
+
} else if (directiveName === 'relationship') {
|
|
139
|
+
const relationshipDefinition = {};
|
|
140
|
+
for (const arg of directive.arguments) {
|
|
141
|
+
relationshipDefinition[arg.name.value] = (arg.value as StringValueNode).value;
|
|
142
|
+
}
|
|
143
|
+
property.relationship = relationshipDefinition;
|
|
144
|
+
} else if (directiveName === 'createdTime') {
|
|
145
|
+
property.assignCreatedTime = true;
|
|
146
|
+
} else if (directiveName === 'updatedTime') {
|
|
147
|
+
property.assignUpdatedTime = true;
|
|
148
|
+
} else if (directiveName === 'expiresAt') {
|
|
149
|
+
property.expiresAt = true;
|
|
150
|
+
} else if (directiveName === 'enumerable') {
|
|
151
|
+
property.enumerable = true;
|
|
152
|
+
} else if (directiveName === 'allow') {
|
|
153
|
+
const authorizedRoles = (property.authorizedRoles = []);
|
|
154
|
+
for (const arg of directive.arguments) {
|
|
155
|
+
if (arg.name.value === 'role') {
|
|
156
|
+
authorizedRoles.push((arg.value as StringValueNode).value);
|
|
157
157
|
}
|
|
158
|
-
} else if (server.knownGraphQLDirectives.includes(directiveName)) {
|
|
159
|
-
console.warn(`@${directiveName} is an unknown directive, at`, directive.loc);
|
|
160
158
|
}
|
|
159
|
+
} else if (server.knownGraphQLDirectives.includes(directiveName)) {
|
|
160
|
+
console.warn(`@${directiveName} is an unknown directive, at`, directive.loc);
|
|
161
161
|
}
|
|
162
162
|
}
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
}
|
|
164
|
+
typeDef.type = typeName;
|
|
165
165
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
166
|
+
}
|
|
167
|
+
// check the types and if any types reference other types, fill those in.
|
|
168
|
+
function connectPropertyType(property) {
|
|
169
|
+
const targetTypeDef = types.get(property.type);
|
|
170
|
+
if (targetTypeDef) {
|
|
171
|
+
Object.defineProperty(property, 'properties', { value: targetTypeDef.properties });
|
|
172
|
+
Object.defineProperty(property, 'definition', { value: targetTypeDef });
|
|
173
|
+
} else if (property.type === 'array') connectPropertyType(property.elements);
|
|
174
|
+
else if (!PRIMITIVE_TYPES.includes(property.type)) {
|
|
175
|
+
if (getWorkerIndex() === 0)
|
|
176
|
+
console.error(
|
|
177
|
+
`The type ${property.type} is unknown at line ${property.location.line}, column ${property.location.column}, in ${filePath}`
|
|
178
|
+
);
|
|
179
179
|
}
|
|
180
|
-
|
|
181
|
-
|
|
180
|
+
}
|
|
181
|
+
for (const typeDef of types.values()) {
|
|
182
|
+
for (const property of typeDef.properties) connectPropertyType(property);
|
|
183
|
+
}
|
|
184
|
+
// any tables that are defined in the schema can now be registered
|
|
185
|
+
for (const typeDef of tables) {
|
|
186
|
+
// with graphql database definitions, this is a declaration that the table should exist and that it
|
|
187
|
+
// should be created if it does not exist
|
|
188
|
+
typeDef.tableClass = table(typeDef);
|
|
189
|
+
if (typeDef.export) {
|
|
190
|
+
// allow empty string to be used to declare a table on the root path
|
|
191
|
+
if (typeDef.export.name === '') resources.set(dirname(urlPath), typeDef.tableClass);
|
|
192
|
+
else
|
|
193
|
+
resources.set(
|
|
194
|
+
dirname(urlPath) + '/' + (typeDef.export.name || typeDef.type),
|
|
195
|
+
typeDef.tableClass,
|
|
196
|
+
typeDef.export
|
|
197
|
+
);
|
|
182
198
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
);
|
|
199
|
+
}
|
|
200
|
+
function createComputedFrom(computedFrom: string, arg: any, attributes: any) {
|
|
201
|
+
// Create a function from a computed "from" directive. This can look like:
|
|
202
|
+
// @computed(from: "fieldOne + fieldTwo")
|
|
203
|
+
// We use Node's built-in Script class to compile the function and run it in the context of the record object, which allows us to specify the source
|
|
204
|
+
const script = new Script(
|
|
205
|
+
// we use the inner with statement to allow the computed function to access the record object's properties directly as top level names
|
|
206
|
+
// we use the outer with statement with attributes as a fallback so any access to an attribute that isn't defined on the record still returns undefined (instead of a ReferenceError)
|
|
207
|
+
`function computed(attributes) { return function(record) { with(attributes) { with (record) { return ${computedFrom}; } } } } computed;`,
|
|
208
|
+
{
|
|
209
|
+
filename: filePath, // specify the file path and line position for better error messages/debugging
|
|
210
|
+
lineOffset: arg.loc.startToken.line - 1,
|
|
211
|
+
columnOffset: arg.loc.startToken.column,
|
|
197
212
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
// Create a function from a computed "from" directive. This can look like:
|
|
201
|
-
// @computed(from: "fieldOne + fieldTwo")
|
|
202
|
-
// We use Node's built-in Script class to compile the function and run it in the context of the record object, which allows us to specify the source
|
|
203
|
-
const script = new Script(
|
|
204
|
-
// we use the inner with statement to allow the computed function to access the record object's properties directly as top level names
|
|
205
|
-
// we use the outer with statement with attributes as a fallback so any access to an attribute that isn't defined on the record still returns undefined (instead of a ReferenceError)
|
|
206
|
-
`function computed(attributes) { return function(record) { with(attributes) { with (record) { return ${computedFrom}; } } } } computed;`,
|
|
207
|
-
{
|
|
208
|
-
filename: filePath, // specify the file path and line position for better error messages/debugging
|
|
209
|
-
lineOffset: arg.loc.startToken.line - 1,
|
|
210
|
-
columnOffset: arg.loc.startToken.column,
|
|
211
|
-
}
|
|
212
|
-
);
|
|
213
|
-
return script.runInThisContext()(attributes); // run the script in the context of the current context/global and return the function we defined
|
|
214
|
-
}
|
|
213
|
+
);
|
|
214
|
+
return script.runInThisContext()(attributes); // run the script in the context of the current context/global and return the function we defined
|
|
215
215
|
}
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
export const startOnMainThread = start;
|
|
219
218
|
// useful for testing
|
|
220
|
-
export const loadGQLSchema = (content) =>
|
|
221
|
-
start({ ensureTable: table }).handleFile(content, null, null, new Resources());
|
|
219
|
+
export const loadGQLSchema = (content) => processGraphQLSchema(content, null, null, new Resources());
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cosineDistance, euclideanDistance } from './vector.ts';
|
|
1
|
+
import { cosineDistance, euclideanDistance, dotProductDistance } from './vector.ts';
|
|
2
2
|
import { FLOAT32_OPTIONS } from 'msgpackr';
|
|
3
3
|
import { loggerWithTag } from '../../utility/logging/logger.ts';
|
|
4
4
|
import { ClientError } from '../../utility/errors/hdbError.js';
|
|
@@ -52,7 +52,12 @@ export class HierarchicalNavigableSmallWorld {
|
|
|
52
52
|
// (we would actually like to use float16 if it were available)
|
|
53
53
|
this.indexStore.encoder.useFloat32 = FLOAT32_OPTIONS.ALWAYS;
|
|
54
54
|
}
|
|
55
|
-
this.distance =
|
|
55
|
+
this.distance =
|
|
56
|
+
options?.distance === 'euclidean'
|
|
57
|
+
? euclideanDistance
|
|
58
|
+
: options?.distance === 'dotProduct'
|
|
59
|
+
? dotProductDistance
|
|
60
|
+
: cosineDistance;
|
|
56
61
|
if (options) {
|
|
57
62
|
// allow all the HNSW parameters to be configured/tuned
|
|
58
63
|
if (options.M !== undefined) {
|
|
@@ -469,6 +474,7 @@ export class HierarchicalNavigableSmallWorld {
|
|
|
469
474
|
let distanceFunction: (a: number[], b: number[]) => number;
|
|
470
475
|
if (distance === 'cosine') distanceFunction = cosineDistance;
|
|
471
476
|
else if (distance === 'euclidean') distanceFunction = euclideanDistance;
|
|
477
|
+
else if (distance === 'dotProduct') distanceFunction = dotProductDistance;
|
|
472
478
|
else if (distance) throw new ClientError('Unknown distance function');
|
|
473
479
|
else distanceFunction = this.distance;
|
|
474
480
|
if (!target) throw new ClientError('A target vector must be provided for an HNSW query');
|
|
@@ -658,7 +664,12 @@ export class HierarchicalNavigableSmallWorld {
|
|
|
658
664
|
|
|
659
665
|
let distanceFunction = this.distance;
|
|
660
666
|
if (sortDefinition.type)
|
|
661
|
-
distanceFunction =
|
|
667
|
+
distanceFunction =
|
|
668
|
+
sortDefinition.distance === 'euclidean'
|
|
669
|
+
? euclideanDistance
|
|
670
|
+
: sortDefinition.distance === 'dotProduct'
|
|
671
|
+
? dotProductDistance
|
|
672
|
+
: cosineDistance;
|
|
662
673
|
const distance = distanceFunction(sortDefinition.target, vector);
|
|
663
674
|
vectorDistances.set(entry, distance);
|
|
664
675
|
return distance;
|
|
@@ -36,3 +36,20 @@ export function cosineDistance(a: number[], b: number[]): number {
|
|
|
36
36
|
|
|
37
37
|
return 1 - dotProduct / (magnitudeA * magnitudeB || 1);
|
|
38
38
|
}
|
|
39
|
+
|
|
40
|
+
export function dotProductDistance(a: number[], b: number[]): number {
|
|
41
|
+
if (!Array.isArray(a) || !Array.isArray(b)) {
|
|
42
|
+
throw new Error('Inner product comparison requires an array');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
let dotProduct = 0;
|
|
46
|
+
const length = Math.max(a.length, b.length);
|
|
47
|
+
|
|
48
|
+
for (let i = 0; i < length; i++) {
|
|
49
|
+
const va = a[i] || 0;
|
|
50
|
+
const vb = b[i] || 0;
|
|
51
|
+
dotProduct += va * vb;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return -dotProduct;
|
|
55
|
+
}
|
|
@@ -1,22 +1,26 @@
|
|
|
1
1
|
import { parse } from 'dotenv';
|
|
2
2
|
import logger from '../utility/logging/harper_logger.js';
|
|
3
|
+
import { Scope } from '../components/Scope.ts';
|
|
3
4
|
|
|
4
|
-
export function
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
5
|
+
export function handleApplication(scope: Scope) {
|
|
6
|
+
const override = (scope.options.getAll() as { override?: boolean }).override ?? false;
|
|
7
|
+
scope.handleEntry((entry) => {
|
|
8
|
+
if (entry.eventType !== 'add') {
|
|
9
|
+
scope.requestRestart();
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
12
|
+
logger.debug(`Loading env file: ${entry.absolutePath}`);
|
|
13
|
+
for (const [key, value] of Object.entries(parse(entry.contents))) {
|
|
14
|
+
if (process.env[key] !== undefined) {
|
|
15
|
+
logger.warn(`Environment variable conflict: ${key} from ${entry.absolutePath} is already set on process.env`);
|
|
16
|
+
if (override) {
|
|
17
|
+
logger.debug(`override option enabled. overriding environment variable: ${key}`);
|
|
18
|
+
} else {
|
|
19
|
+
continue;
|
|
16
20
|
}
|
|
17
|
-
|
|
18
|
-
process.env[key] = value;
|
|
19
21
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
|
|
23
|
+
process.env[key] = value;
|
|
24
|
+
}
|
|
25
|
+
});
|
|
22
26
|
}
|
package/core/resources/login.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Resource } from './Resource.ts';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
resources.
|
|
2
|
+
import { Scope } from '../components/Scope.ts';
|
|
3
|
+
export function handleApplication(scope: Scope) {
|
|
4
|
+
scope.resources.set('login', Login);
|
|
5
|
+
scope.resources.loginPath = (request) => {
|
|
5
6
|
return '/login?redirect=' + encodeURIComponent(request.url);
|
|
6
7
|
};
|
|
7
8
|
}
|