@harperfast/harper-pro 5.0.30 → 5.1.0-beta.2

package/core/resources/graphql.ts

@@ -4,6 +4,10 @@ import { table } from './databases.ts';
 import { getWorkerIndex } from '../server/threads/manageThreads.js';
 import { Resources } from './Resources.ts';
 import type { NamedTypeNode, StringValueNode } from 'graphql';
+import { once } from 'node:events';
+import { ClientError } from '../utility/errors/hdbError.ts';
+import { attributeToFragment, type JsonSchemaFragment } from './jsonSchemaTypes.ts';
+import harperLogger from '../utility/logging/harper_logger.ts';
 
 const PRIMITIVE_TYPES = ['ID', 'Int', 'Float', 'Long', 'String', 'Boolean', 'Date', 'Bytes', 'Any', 'BigInt', 'Blob'];
 
@@ -17,12 +21,14 @@ server.knownGraphQLDirectives.push(
 	'primaryKey',
 	'indexed',
 	'computed',
+	'embed',
 	'relationship',
 	'createdTime',
 	'updatedTime',
 	'expiresAt',
 	'allow',
-	'enumerable'
+	'enumerable',
+	'hidden'
 );
 /**
  * This is the entry point for handling GraphQL schemas (and server-side defined queries, eventually). This will be
@@ -35,187 +41,283 @@ server.knownGraphQLDirectives.push(
  * @param filePath
  * @param resources
  */
-export function start({ ensureTable }) {
-	return {
-		handleFile,
-		setupFile: handleFile,
-	};
+export function handleApplication(scope: import('../components/Scope.ts').Scope) {
+	let initialLoadComplete = false;
+	const entryHandler = scope.handleEntry(async (entry) => {
+		if (initialLoadComplete) {
+			scope.requestRestart();
+			return;
+		}
 
-	async function handleFile(gqlContent, urlPath, filePath, resources) {
-		// lazy load the graphql package so we don't load it for users that don't use graphql
-		const { parse, Source, Kind } = await import('graphql');
-		const ast = parse(new Source(gqlContent.toString(), filePath));
-		const types = new Map();
-		const tables = [];
-		// we begin by iterating through the definitions in the AST to get the types and convert them
-		// to a friendly format for table attributes
-		for (const definition of ast.definitions) {
-			switch (definition.kind) {
-				case Kind.OBJECT_TYPE_DEFINITION:
-					const typeName = definition.name.value;
-					// use type name as the default table
-					const properties = [];
-					const typeDef = { table: null, database: null, properties };
-					types.set(typeName, typeDef);
-					resources.allTypes.set(typeName, typeDef);
-					for (const directive of definition.directives) {
-						const directiveName = directive.name.value;
-						if (directiveName === 'table') {
-							for (const arg of directive.arguments) {
-								typeDef[arg.name.value] = (arg.value as StringValueNode).value;
-							}
-							if (typeDef.schema) typeDef.database = typeDef.schema;
-							if (!typeDef.table) typeDef.table = typeName;
-							if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
-							typeDef.attributes = typeDef.properties;
-							tables.push(typeDef);
-						}
-						if (directive.name.value === 'sealed') typeDef.sealed = true;
-						if (directive.name.value === 'splitSegments') typeDef.splitSegments = true;
-						if (directive.name.value === 'replicate') typeDef.replicate = true;
-						if (directive.name.value === 'export') {
-							typeDef.export = true;
-							for (const arg of directive.arguments) {
-								if (typeof typeDef.export !== 'object') typeDef.export = {};
-								typeDef.export[arg.name.value] = (arg.value as StringValueNode).value;
-							}
+		if (entry.eventType === 'unlink') return;
+		if (entry.entryType === 'directory') {
+			scope.logger.warn?.('graphqlSchema currently does not handle directories. Specify file patterns only.');
+			return;
+		}
+
+		await processGraphQLSchema((entry as any).contents, entry.urlPath, entry.absolutePath, scope.resources);
+	});
+	const initialLoadPromise = once(entryHandler, 'initialLoadComplete');
+	initialLoadPromise.then(() => {
+		initialLoadComplete = true;
+	});
+	return initialLoadPromise;
+}
+
+async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
+	// lazy load the graphql package so we don't load it for users that don't use graphql
+	const { parse, Source, Kind } = await import('graphql');
+	const ast = parse(new Source(gqlContent.toString(), filePath));
+	const types = new Map();
+	const tables = [];
+	// we begin by iterating through the definitions in the AST to get the types and convert them
+	// to a friendly format for table attributes
+	for (const definition of ast.definitions) {
+		switch (definition.kind) {
+			case Kind.OBJECT_TYPE_DEFINITION:
+				const typeName = definition.name.value;
+				// use type name as the default table
+				const attributes: any[] = [];
+				const typeProperties: Record<string, JsonSchemaFragment> = {};
+				const typeDef: any = { table: null, database: null, attributes, properties: typeProperties };
+				if (definition.description?.value) typeDef.description = definition.description.value;
+				types.set(typeName, typeDef);
+				resources.allTypes.set(typeName, typeDef);
+				for (const directive of definition.directives) {
+					const directiveName = directive.name.value;
+					if (directiveName === 'table') {
+						for (const arg of directive.arguments) {
+							typeDef[arg.name.value] = (arg.value as StringValueNode).value;
 						}
+						if (typeDef.schema) typeDef.database = typeDef.schema;
+						if (!typeDef.table) typeDef.table = typeName;
+						if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
+						// Boolean directive args arrive as actual booleans; tolerate string forms too.
+						if (typeDef.randomAccessFields !== undefined)
+							typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true';
+						tables.push(typeDef);
 					}
-					let hasPrimaryKey = false;
-					function getProperty(type) {
-						if (type.kind === 'NonNullType') {
-							const property = getProperty(type.type);
-							property.nullable = false;
-							return property;
+					if (directive.name.value === 'sealed') typeDef.sealed = true;
+					if (directive.name.value === 'splitSegments') typeDef.splitSegments = true;
+					if (directive.name.value === 'replicate') typeDef.replicate = true;
+					if (directive.name.value === 'hidden') typeDef.hidden = true;
+					if (directive.name.value === 'export') {
+						typeDef.export = true;
+						for (const arg of directive.arguments) {
+							if (typeof typeDef.export !== 'object') typeDef.export = {};
+							typeDef.export[arg.name.value] = (arg.value as StringValueNode).value;
 						}
-						if (type.kind === 'ListType') {
-							return {
-								type: 'array',
-								elements: getProperty(type.type),
-							};
-						}
-						const typeName = (type as NamedTypeNode).name?.value;
-						const property = { type: typeName };
-						Object.defineProperty(property, 'location', { value: type.loc.startToken });
+					}
+				}
+				let hasPrimaryKey = false;
+				function getProperty(type) {
+					if (type.kind === 'NonNullType') {
+						const property = getProperty(type.type);
+						property.nullable = false;
 						return property;
 					}
-					const attributesObject = {};
-					for (const field of definition.fields) {
-						const property = getProperty(field.type);
-						property.name = field.name.value;
-						properties.push(property);
-						attributesObject[property.name] = undefined; // this is used as a backup scope for computed properties
-						for (const directive of field.directives) {
-							const directiveName = directive.name.value;
-							if (directiveName === 'primaryKey') {
-								if (hasPrimaryKey) console.warn('Can not define two attributes as a primary key at', directive.loc);
-								else {
-									property.isPrimaryKey = true;
-									hasPrimaryKey = true;
-								}
-							} else if (directiveName === 'indexed') {
-								const indexedDefinition = {};
-								// store indexed arguments for configurable indexes.
-								for (const arg of directive.arguments || []) {
-									indexedDefinition[arg.name.value] = (arg.value as StringValueNode).value;
-								}
-								property.indexed = indexedDefinition;
-							} else if (directiveName === 'computed') {
-								for (const arg of directive.arguments || []) {
-									if (arg.name.value === 'from') {
-										const computedFromExpression = (arg.value as StringValueNode).value;
-										property.computed = {
-											from: createComputedFrom(computedFromExpression, arg, attributesObject),
-										};
-										// if the version is not defined, we use the computed from expression as the version, any changes to the computed from expression will trigger a version change and reindex
-										if (property.version == undefined) property.version = computedFromExpression;
-									} else if (arg.name.value === 'version') {
-										property.version = (arg.value as StringValueNode).value;
-									}
+					if (type.kind === 'ListType') {
+						return {
+							type: 'array',
+							elements: getProperty(type.type),
+						};
+					}
+					const typeName = (type as NamedTypeNode).name?.value;
+					const property = { type: typeName };
+					Object.defineProperty(property, 'location', { value: type.loc.startToken });
+					return property;
+				}
+				const attributesObject = {};
+				for (const field of definition.fields) {
+					const property = getProperty(field.type);
+					property.name = field.name.value;
+					if (field.description?.value) property.description = field.description.value;
+					attributes.push(property);
+					attributesObject[property.name] = undefined; // this is used as a backup scope for computed properties
+					for (const directive of field.directives) {
+						const directiveName = directive.name.value;
+						if (directiveName === 'primaryKey') {
+							if (hasPrimaryKey) console.warn('Can not define two attributes as a primary key at', directive.loc);
+							else {
+								property.isPrimaryKey = true;
+								hasPrimaryKey = true;
+							}
+						} else if (directiveName === 'indexed') {
+							const indexedDefinition = {};
+							// store indexed arguments for configurable indexes.
+							for (const arg of directive.arguments || []) {
+								indexedDefinition[arg.name.value] = (arg.value as StringValueNode).value;
+							}
+							property.indexed = indexedDefinition;
+						} else if (directiveName === 'computed') {
+							for (const arg of directive.arguments || []) {
+								if (arg.name.value === 'from') {
+									const computedFromExpression = (arg.value as StringValueNode).value;
+									property.computedFromExpression = computedFromExpression;
+									property.computed = {
+										from: createComputedFrom(computedFromExpression, arg, attributesObject),
+									};
+									// if the version is not defined, we use the computed from expression as the version, any changes to the computed from expression will trigger a version change and reindex
+									if (property.version == undefined) property.version = computedFromExpression;
+								} else if (arg.name.value === 'version') {
+									property.version = (arg.value as StringValueNode).value;
 								}
-								property.computed = property.computed || true;
-							} else if (directiveName === 'relationship') {
-								const relationshipDefinition = {};
-								for (const arg of directive.arguments) {
-									relationshipDefinition[arg.name.value] = (arg.value as StringValueNode).value;
+							}
+							property.computed = property.computed || true;
+						} else if (directiveName === 'embed') {
+							// `@embed(source, model)`: on write, embed `record[source]` into this
+							// attribute and auto-index it with HNSW.
+							const embedDefinition: { source?: string; model?: string } = {};
+							for (const arg of directive.arguments || []) {
+								if (arg.value.kind !== 'StringValue')
+									throw new ClientError(
+										`@embed(${arg.name.value}: ...) on "${property.name}" expects a string literal`,
+										400
+									);
+								embedDefinition[arg.name.value] = (arg.value as StringValueNode).value;
+							}
+							if (!embedDefinition.source || !embedDefinition.model) {
+								const loc = directive.loc;
+								throw new ClientError(
+									`@embed on "${property.name}" requires both "source" and "model" arguments` +
+										(loc ? ` (line ${loc.startToken?.line ?? '?'}, column ${loc.startToken?.column ?? '?'})` : ''),
+									400
+								);
+							} else {
+								property.embed = embedDefinition;
+								// Version carries the model so a model change triggers a reindex (re-index only, not re-embed).
+								if (property.version == undefined) {
+									property.version = `embed:${embedDefinition.model}`;
 								}
-								property.relationship = relationshipDefinition;
-							} else if (directiveName === 'createdTime') {
-								property.assignCreatedTime = true;
-							} else if (directiveName === 'updatedTime') {
-								property.assignUpdatedTime = true;
-							} else if (directiveName === 'expiresAt') {
-								property.expiresAt = true;
-							} else if (directiveName === 'enumerable') {
-								property.enumerable = true;
-							} else if (directiveName === 'allow') {
-								const authorizedRoles = (property.authorizedRoles = []);
-								for (const arg of directive.arguments) {
-									if (arg.name.value === 'role') {
-										authorizedRoles.push((arg.value as StringValueNode).value);
-									}
+							}
+						} else if (directiveName === 'relationship') {
+							const relationshipDefinition = {};
+							for (const arg of directive.arguments) {
+								relationshipDefinition[arg.name.value] = (arg.value as StringValueNode).value;
+							}
+							property.relationship = relationshipDefinition;
+						} else if (directiveName === 'createdTime') {
+							property.assignCreatedTime = true;
+						} else if (directiveName === 'updatedTime') {
+							property.assignUpdatedTime = true;
+						} else if (directiveName === 'expiresAt') {
+							property.expiresAt = true;
+						} else if (directiveName === 'enumerable') {
+							property.enumerable = true;
+						} else if (directiveName === 'hidden') {
+							property.hidden = true;
+						} else if (directiveName === 'allow') {
+							const authorizedRoles = (property.authorizedRoles = []);
+							for (const arg of directive.arguments) {
+								if (arg.name.value === 'role') {
+									authorizedRoles.push((arg.value as StringValueNode).value);
 								}
-							} else if (server.knownGraphQLDirectives.includes(directiveName)) {
-								console.warn(`@${directiveName} is an unknown directive, at`, directive.loc);
 							}
+						} else if (server.knownGraphQLDirectives.includes(directiveName)) {
+							console.warn(`@${directiveName} is an unknown directive, at`, directive.loc);
 						}
 					}
-					typeDef.type = typeName;
-			}
-		}
-		// check the types and if any types reference other types, fill those in.
-		function connectPropertyType(property) {
-			const targetTypeDef = types.get(property.type);
-			if (targetTypeDef) {
-				Object.defineProperty(property, 'properties', { value: targetTypeDef.properties });
-				Object.defineProperty(property, 'definition', { value: targetTypeDef });
-			} else if (property.type === 'array') connectPropertyType(property.elements);
-			else if (!PRIMITIVE_TYPES.includes(property.type)) {
-				if (getWorkerIndex() === 0)
-					console.error(
-						`The type ${property.type} is unknown at line ${property.location.line}, column ${property.location.column}, in ${filePath}`
-					);
-			}
+					// @embed targets a vector column and auto-indexes it with HNSW; resolved after all
+					// directives so an explicit @indexed (in any order) is honored. The target must be an
+					// array (e.g. [Float]) — a scalar would store the vector wrong and HNSW-index a non-vector.
+					if (property.embed) {
+						const elementType =
+							property.type === 'array' ? (property as { elements?: { type?: string } }).elements?.type : undefined;
+						if (property.type !== 'array' || elementType !== 'Float')
+							throw new ClientError(
+								`@embed on "${property.name}" requires a [Float] attribute type; got "${property.type === 'array' ? `[${elementType ?? '?'}]` : property.type}"`,
+								400
+							);
+						if (!property.indexed) property.indexed = { type: 'HNSW' };
+						else if ((property.indexed as { type?: string }).type !== 'HNSW')
+							throw new ClientError(
+								`@embed on "${property.name}" auto-indexes with HNSW; remove the conflicting @indexed or set @indexed(type: "HNSW")`,
+								400
+							);
+					}
+				}
+				// @embed source must reference a declared field; a typo would silently leave
+				// the vector column unpopulated (the source key never appears in write payloads).
+				for (const prop of attributes as any[]) {
+					// Object.hasOwn (not `in`): `attributesObject` is a plain object, so `in` would
+					// match inherited prototype keys (toString, constructor) and pass a bad source.
+					if (prop.embed && !Object.hasOwn(attributesObject, prop.embed.source))
+						throw new ClientError(
+							`@embed on "${prop.name}" references unknown source field "${prop.embed.source}"`,
+							400
+						);
+				}
+				// Project the array form into the canonical `properties` Record (JSON-Schema-shaped,
+				// keyed by attribute name). Both shapes are co-populated in this single pass;
+				// downstream consumers (MCP, OpenAPI) read whichever form they prefer.
+				// `attributeToFragment` handles array types recursively so primitive arrays
+				// (e.g. `[String]`) emit `{ type: 'array', items: { type: 'string' } }`.
+				for (const prop of attributes as any[]) {
+					typeProperties[prop.name] = attributeToFragment(prop);
+				}
+				typeDef.type = typeName;
 		}
-		for (const typeDef of types.values()) {
-			for (const property of typeDef.properties) connectPropertyType(property);
+	}
+	// check the types and if any types reference other types, fill those in.
+	function connectPropertyType(property) {
+		const targetTypeDef = types.get(property.type);
+		if (targetTypeDef) {
+			// `property.properties` on a complex-type attribute carries the nested Array of
+			// sub-attributes (Attribute.properties — Array<Attribute>). Keep reading from
+			// `targetTypeDef.attributes` (the internal Array form) rather than the new
+			// class-level `typeDef.properties` (the Record canonical surface).
+			Object.defineProperty(property, 'properties', { value: targetTypeDef.attributes });
+			Object.defineProperty(property, 'definition', { value: targetTypeDef });
+		} else if (property.type === 'array') connectPropertyType(property.elements);
+		else if (!PRIMITIVE_TYPES.includes(property.type)) {
+			if (getWorkerIndex() === 0)
+				console.error(
+					`The type ${property.type} is unknown at line ${property.location.line}, column ${property.location.column}, in ${filePath}`
+				);
 		}
-		// any tables that are defined in the schema can now be registered
-		for (const typeDef of tables) {
-			// with graphql database definitions, this is a declaration that the table should exist and that it
-			// should be created if it does not exist
-			typeDef.tableClass = ensureTable(typeDef);
-			if (typeDef.export) {
-				// allow empty string to be used to declare a table on the root path
-				if (typeDef.export.name === '') resources.set(dirname(urlPath), typeDef.tableClass);
-				else
-					resources.set(
-						dirname(urlPath) + '/' + (typeDef.export.name || typeDef.type),
-						typeDef.tableClass,
-						typeDef.export
-					);
-			}
+	}
+	for (const typeDef of types.values()) {
+		for (const property of typeDef.attributes) connectPropertyType(property);
+	}
+	// any tables that are defined in the schema can now be registered
+	for (const typeDef of tables) {
+		// with graphql database definitions, this is a declaration that the table should exist and that it
+		// should be created if it does not exist
+		typeDef.tableClass = table(typeDef);
+		if (getWorkerIndex() === 0) {
+			// Post-Phase-2: typeDef.properties is the canonical Record (no .find); read the Array form.
+			const pk = (typeDef.attributes as any[])?.find((p) => p.isPrimaryKey)?.name ?? 'id';
+			const schemaPart = typeDef.database ? `, schema: ${typeDef.database}` : '';
+			harperLogger.info?.(`Initialized table "${typeDef.table}"${schemaPart}, primaryKey: ${pk}`);
 		}
-		function createComputedFrom(computedFrom: string, arg: any, attributes: any) {
-			// Create a function from a computed "from" directive. This can look like:
-			// @computed(from: "fieldOne + fieldTwo")
-			// We use Node's built-in Script class to compile the function and run it in the context of the record object, which allows us to specify the source
-			const script = new Script(
-				// we use the inner with statement to allow the computed function to access the record object's properties directly as top level names
-				// we use the outer with statement with attributes as a fallback so any access to an attribute that isn't defined on the record still returns undefined (instead of a ReferenceError)
-				`function computed(attributes) { return function(record) { with(attributes) { with (record) { return ${computedFrom}; } } } } computed;`,
-				{
-					filename: filePath, // specify the file path and line position for better error messages/debugging
-					lineOffset: arg.loc.startToken.line - 1,
-					columnOffset: arg.loc.startToken.column,
-				}
-			);
-			return script.runInThisContext()(attributes); // run the script in the context of the current context/global and return the function we defined
+		if (typeDef.export) {
+			// allow empty string to be used to declare a table on the root path
+			if (typeDef.export.name === '') resources.set(dirname(urlPath), typeDef.tableClass);
+			else
+				resources.set(
+					dirname(urlPath) + '/' + (typeDef.export.name || typeDef.type),
+					typeDef.tableClass,
+					typeDef.export
+				);
 		}
 	}
+	function createComputedFrom(computedFrom: string, arg: any, attributes: any) {
+		// Create a function from a computed "from" directive. This can look like:
+		// @computed(from: "fieldOne + fieldTwo")
+		// We use Node's built-in Script class to compile the function and run it in the context of the record object, which allows us to specify the source
+		const script = new Script(
+			// we use the inner with statement to allow the computed function to access the record object's properties directly as top level names
+			// we use the outer with statement with attributes as a fallback so any access to an attribute that isn't defined on the record still returns undefined (instead of a ReferenceError)
+			`function computed(attributes) { return function(record) { with(attributes) { with (record) { return ${computedFrom}; } } } } computed;`,
+			{
+				filename: filePath, // specify the file path and line position for better error messages/debugging
+				lineOffset: arg.loc.startToken.line - 1,
+				columnOffset: arg.loc.startToken.column,
+			}
+		);
+		return script.runInThisContext()(attributes); // run the script in the context of the current context/global and return the function we defined
+	}
 }
 
-export const startOnMainThread = start;
 // useful for testing
-export const loadGQLSchema = (content) =>
-	start({ ensureTable: table }).handleFile(content, null, null, new Resources());
+export const loadGQLSchema = (content) => processGraphQLSchema(content, null, null, new Resources());