@sap/cds-compiler 5.0.6 → 5.1.2

package/CHANGELOG.md

@@ -7,6 +7,37 @@
 Note: `beta` fixes, changes and features are usually not listed in this ChangeLog but [here](doc/CHANGELOG_BETA.md).
 The compiler behavior concerning `beta` features can change at any time without notice.
 
+## Version 5.1.2 - 2024-08-05
+
+### Fixed
+
+- compiler: In parseCdl mode, bound actions specifying the binding parameter with `$self` did not work.
+
+## Version 5.1.0 - 2024-07-25
+
+### Added
+
+- cdsc: Option `--stdin` was added to support input via standard input, e.g. `cat file.cds | cdsc --stdin`
+- Allow to refer to draft state element `IsActiveEntity` via magic variable `$draft.IsActiveEntity` in annotation path expressions.
+  + for.odata: During draft augmentation `$draft.IsActiveEntity` is rewritten to `$self.IsActiveEntity` for all draft enabled
+    entities (root and sub nodes but not for named types or entity parameters).
+  + to.edm(x): (V4 only) Allow to refer to an entity element in a bound action via `$self` and not only via explicit binding parameter
+    in an annotation path expression. The API generator will prefix the path with the actual binding parameter name (explicit, annotation or
+    default).
+
+### Changed
+
+- Update OData vocabularies: 'Common', 'Core', 'HTML5', 'UI'.
+- to.cdl|hdbcds|hdi|sql: Remove `generated by` comment.
+
+### Fixed
+
+- compiler: checks for associations now work for nested projections of the form `association.{ id }`
+- to.edm(x): No `Nullable` attribute for `$ReturnType` of `Collection(<entity type>)` [OData V4 CSDL, section 12.8 Return Type](https://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/odata-csdl-xml-v4.01.html#sec_ReturnType)
+- to.sql|hdi|hdbcds: Detect and error on "cross-eyed" backlinks, where we cannot construct a valid on-condition.
+- to.sql|hdi.migration: Correctly detect that a view was dropped - this was previously just silently ignored.
+
+
 ## Version 5.0.6 - 2024-07-10
 
 ### Fixed
@@ -73,6 +104,16 @@ This is a preview version for the major release and contains breaking changes. I
 - API: Deprecated functions `preparedCsnToEdmx` and `preparedCsnToEdm` were removed.
   Use `to.edm(x)` instead.
 
+
+## Version 4.9.6 - 2024-07-15
+
+### Fixed
+
+- for.seal: Don't generate DRAFT artifacts.
+- for.odata: Propagate all `@odata { Type, MaxLength, Precision, Scale, SRID }` to generated foreign keys.
+- to.edm(x): Respect `AppliesTo` specification in term definitions for actions and functions.
+- to.sql: Conditions inside filters in combination with foreign key aliases were not properly translated in rare cases.
+
 ## Version 4.9.4 - 2024-05-21
 
 ### Fixed

package/bin/cdsc.js

@@ -35,6 +35,7 @@ const { addLocalizationViews } = require('../lib/transform/localized');
 const { addTenantFields } = require('../lib/transform/addTenantFields');
 const { availableBetaFlags } = require('../lib/base/model');
 const { alterConstraintsWithCsn } = require('../lib/render/manageConstraints');
+const { tmpFilePath, readStream } = require('../lib/utils/file');
 
 // Note: Instead of throwing ProcessExitError, we would rather just call process.exit(exitCode),
 // but that might truncate the output of stdout and stderr, both of which are async (or rather,
@@ -85,9 +86,12 @@ function remapCmdOptions( options, command ) {
 }
 
 function cdsc_main() {
-  // Parse the command line and translate it into options
+  if (process.argv.some(arg => arg === '-i' || arg === '--stdin'))
+    optionProcessor.makePositionalArgumentsOptional();
 
+  // Parse the command line and translate it into options
   const cmdLine = optionProcessor.processCmdLine(process.argv);
+
   // Deal with '--version' explicitly
   if (cmdLine.options.version) {
     process.stdout.write(`${main.version()}\n`);
@@ -210,6 +214,9 @@ function cdsc_main() {
   if (typeof cmdLine.options.moduleLookupDirectories === 'string')
     cmdLine.options.moduleLookupDirectories = cmdLine.options.moduleLookupDirectories.split(',');
 
+  if (cmdLine.options.stdin)
+    cmdLine.options.fallbackParser ??= 'auto!';
+
   parseSeverityOptions(cmdLine);
 
   // Do the work for the selected command
@@ -230,12 +237,12 @@ function validateDirectBackendOption( command, options, args ) {
     displayUsage(`Option '--direct-backend' can't be used with command '${command}'`,
                  optionProcessor.helpText, 2);
   }
-  if (!args.files || args.files.length !== 1) {
-    displayUsage(`Option '--direct-backend' expects exactly one JSON file, but ${args.files.length} given`,
+  if (!options.stdin && (!args.files || args.files.length !== 1)) {
+    displayUsage(`Option '--direct-backend' expects exactly one JSON file, but ${args.files?.length || 'none'} given`,
                  optionProcessor.helpText, 2);
   }
-  const filename = args.files[0];
-  if (!filename.endsWith('.csn') && !filename.endsWith('.json')) {
+  const filename = args.files?.[0];
+  if (filename && !filename.endsWith('.csn') && !filename.endsWith('.json')) {
     displayUsage('Option \'--direct-backend\' expects a filename with a *.csn or *.json suffix',
                  optionProcessor.helpText, 2);
   }
@@ -259,8 +266,21 @@ function displayUsage( error, helpText, code ) {
   throw new ProcessExitError(code);
 }
 
+/**
+ * As the compiler is file-based and will always at least try to call `realpath()` on a file,
+ * we fake a "stdin" file for it.
+ *
+ * @returns {Promise<string>}
+ */
+async function createTemporaryFileFromStdin() {
+  const contents = await readStream(process.stdin);
+  const file = tmpFilePath('cds-compiler-stdin', 'cds');
+  await fs.promises.writeFile(file, contents);
+  return file;
+}
+
 // Executes a command line that has been translated to 'command' (what to do), 'options' (how) and 'args' (which files)
-function executeCommandLine( command, options, args ) {
+async function executeCommandLine( command, options, args ) {
   const normalizeFilename = options.testMode && process.platform === 'win32';
   const messageLevels = {
     Error: 0, Warning: 1, Info: 2, Debug: 3,
@@ -304,17 +324,22 @@ function executeCommandLine( command, options, args ) {
   }
 
   options.messages = [];
+  args.files ??= [];
+
+  // Load a file from stdin if no explicit file is given and stdin is not a TTY.
+  if (options.stdin)
+    args.files.push(await createTemporaryFileFromStdin());
 
   const fileCache = Object.create(null);
   const compiled = options.directBackend
     ? util.promisify(fs.readFile)( args.files[0], 'utf-8' ).then(str => JSON.parse( str ))
     : compiler.compileX( args.files, undefined, options, fileCache );
 
-  compiled.then( commands[command] )
+  await compiled.then( commands[command] )
     .then( displayMessages, displayErrors )
     .catch( catchErrors );
 
-  return; // below are only command implementations.
+  // below are only command implementations.
 
   // Execute the command line option '--to-cdl' and display the results.
   // Return the original model (for chaining)
@@ -724,3 +749,4 @@ function parseSeverityOptions({ options }) {
     }
   }
 }
+

package/bin/cdshi.js

@@ -13,7 +13,8 @@
 
 const compiler = require('../lib/compiler');
 const fs = require('fs');
-fs.readFile( '/dev/stdin', 'utf8', highlight );
+const stdinFd = 0;
+fs.readFile( stdinFd, 'utf8', highlight );
 
 const categoryChars = {         // default: first char of category name
   // first char lowercase = reference other than via extend/annotate:

package/lib/api/main.js

@@ -596,6 +596,15 @@ function sqlMigration( csn, options, messageFunctions, beforeImage ) {
   if (constraintDeletions)
     Object.values(constraintDeletions).forEach(constraint => dropSqls.push(constraint));
 
+  if (Object.keys(drops.final).length > 0) {
+    const order = sortViews({ sql: {}, csn: beforeImage });
+
+    for (const { name } of order) {
+      if (drops.final[name])
+        dropSqls.push(drops.final[name]);
+    }
+  }
+
   // We need to drop the things without dependants first - so inversely sorted
   dropSqls.reverse();
 
@@ -679,7 +688,7 @@ function createSqlDefinitions( hdbkinds, afterImage ) {
  */
 function createSqlDeletions( deletions, beforeImage ) {
   const result = [];
-  objectUtils.forEach(deletions, name => result.push({ name: getFileName(name, beforeImage), suffix: '.hdbtable' }));
+  objectUtils.forEach(deletions, name => result.push({ name: getFileName(name, beforeImage), suffix: beforeImage.definitions[name].query ? '.hdbview' : '.hdbtable' }));
   return result;
 }
 /**

package/lib/api/options.js

@@ -29,7 +29,6 @@ const publicOptionsNewAPI = [
   'magicVars', // deprecated, not removed in v3 as we have specific error messages for it
   'variableReplacements',
   'pre2134ReferentialConstraintNames',
-  'generatedByComment',
   'betterSqliteSessionVariables',
   'fewerLocalizedViews',
   'withHanaAssociations',
@@ -156,7 +155,7 @@ module.exports = {
     sql: (options) => {
       const hardOptions = { src: 'sql', toSql: true, forHana: true };
       const defaultOptions = {
-        sqlMapping: 'plain', sqlDialect: 'plain', generatedByComment: true, withHanaAssociations: true,
+        sqlMapping: 'plain', sqlDialect: 'plain', withHanaAssociations: true,
       };
       const processed = translateOptions(options, defaultOptions, hardOptions, undefined, [ 'sql-dialect-and-naming' ], 'to.sql');
 
@@ -166,7 +165,7 @@ module.exports = {
       const hardOptions = { src: 'hdi', toSql: true, forHana: true };
       // TODO: sqlDialect should be a hard option!
       const defaultOptions = {
-        sqlMapping: 'plain', sqlDialect: 'hana', generatedByComment: false, withHanaAssociations: true,
+        sqlMapping: 'plain', sqlDialect: 'hana', withHanaAssociations: true,
       };
       return translateOptions(options, defaultOptions, hardOptions, { sqlDialect: generateStringValidator([ 'hana' ]) }, undefined, 'to.hdi');
     },

package/lib/base/message-registry.js

@@ -97,6 +97,7 @@ const centralMessages = {
   'ref-expecting-foreign-key': { severity: 'Error' },
   'ref-invalid-source': { severity: 'Error' },
   'ref-invalid-target': { severity: 'Error' },
+  'ref-missing-self-counterpart': { severity: 'Error', configurableFor: true },
   'ref-sloppy-target': { severity: 'Error', configurableFor: 'v4' },
 
   'extend-repeated-intralayer': { severity: 'Warning' },
@@ -556,6 +557,10 @@ const centralMessageTexts = {
   // Messages for erroneous references -----------------------------------------
   // location at erroneous reference (if possible)
   'ref-deprecated-orderby': 'Replace source element reference $(ID) by $(NEWCODE); auto-corrected',
+  'ref-missing-self-counterpart' : {
+    std: 'Expected to find a matching element in $self-comparison for foreign key $(PROP) of association $(NAME)',
+    unmanaged: 'Expected to find a matching element in $self-comparison for $(PROP) of association $(NAME)'
+  },
   'ref-unexpected-self': {
     std: 'Unexpected $(ID) reference; is valid only in ON-conditions of unmanaged associations',
     on: 'Unexpected $(ID) reference; is valid only if compared to be equal to an association of the target side',
@@ -965,6 +970,15 @@ const centralMessageTexts = {
     'managed': 'Ignoring managed association $(NAME) that is published in a UNION',
     'std': 'Ignoring association $(NAME) that is published in a UNION'
   },
+  'query-missing-element': {
+    std: 'Element $(ID) is missing in specified elements',
+    enum: 'Enum $(ID) is missing in specified enum values',
+    foreignKeys: 'Foreign key $(ID) is missing in specified foreign keys',
+  },
+  'query-unspecified-element': {
+    std: 'Element $(ID) does not result from the query',
+    foreignKeys: 'Foreign key $(ID) does not result from the query',
+  },
 
   'ref-sloppy-target': 'An entity or an aspect (not type) is expected here',
 

package/lib/base/messages.js

@@ -13,6 +13,7 @@ const { analyseCsnPath, traverseQuery } = require('../model/csnRefs');
 const { CompilerAssertion } = require('./error');
 const { getArtifactName } = require('../compiler/base');
 const { cdlNewLineRegEx } = require('../language/textUtils');
+const meta = require('./meta');
 
 const fs = require('fs');
 const path = require('path');
@@ -117,7 +118,7 @@ class CompilationError extends Error {
     // no proper message about _what_ the root cause of the exception was.
     // To mitigate that, we serialize the first error in the message as well.
     const firstError = messages.find( m => m.severity === 'Error' )?.toString() || '';
-    super( `CDS compilation failed\n${firstError}` );
+    super( `CDS compilation failed (@sap/cds-compiler v${ meta.version() })\n${firstError}` );
 
     /** @since v4.0.0 */
     this.code = 'ERR_CDS_COMPILATION_FAILURE';

package/lib/base/meta.js

@@ -0,0 +1,10 @@
+'use strict';
+
+// Metadata, e.g. version.
+
+/** The compiler version (taken from package.json) */
+function version() {
+  return require('../../package.json').version;
+}
+
+module.exports = { version };

package/lib/base/optionProcessorHelper.js

@@ -42,6 +42,7 @@ function createOptionProcessor() {
     },
     help,
     processCmdLine,
+    makePositionalArgumentsOptional,
   };
   return optionProcessor;
 
@@ -291,6 +292,16 @@ function createOptionProcessor() {
     };
   }
 
+  function makePositionalArgumentsOptional() {
+    for (const arg of optionProcessor.positionalArguments || [])
+      arg.required = false;
+
+    for (const cmd in optionProcessor.commands) {
+      for (const arg of optionProcessor.commands[cmd].positionalArguments || [])
+        arg.required = false;
+    }
+  }
+
   // API: Let the option processor digest a command line 'argv'
   // The expectation is to get a commandline like this:
   //       $ node cdsc.js -x 1 --foo toXyz -y --bar-wiz bla arg1 arg2

package/lib/checks/dbFeatureFlags.js

@@ -2,6 +2,7 @@
 
 const { setProp } = require('../base/model');
 const { featureFlags } = require('../transform/db/featureFlags');
+const { isSqlService } = require('../transform/db/processSqlServices');
 
 /**
  *
@@ -25,4 +26,8 @@ module.exports = {
   value: setFeatureFlag('$calculatedElements'),
   expand: setFeatureFlag('$expandInline'),
   inline: setFeatureFlag('$expandInline'),
+  kind: function setFeatureFlagForSqlService( artifact ) {
+    if (isSqlService(artifact))
+      setFeatureFlag( '$sqlService' ).call(this);
+  },
 };

package/lib/compiler/assert-consistency.js

@@ -255,6 +255,7 @@ function assertConsistency( model, stage ) {
     elements: { kind: true, inherits: 'definitions', also: [ 0 ] }, // 0 for cyclic expansions
     // specified elements in query entities (TODO: introduce real "specified elements" instead):
     elements$: { kind: true, enumerable: false, test: TODO },
+    foreignKeys$: { kind: true, enumerable: false, test: TODO },
     enum$: { kind: true, enumerable: false, test: TODO },
     typeProps$: { kind: true, enumerable: false, test: TODO },
     // helper property for faster processing:

package/lib/compiler/define.js

@@ -216,7 +216,7 @@ function define( model ) {
 
     const { $self } = model.definitions;
     if ($self) {
-      message( 'name-deprecated-$self', [ $self.location, $self ], { name: '$self' },
+      message( 'name-deprecated-$self', [ $self.name.location, $self ], { name: '$self' },
                'Do not use $(NAME) as name for an artifact definition' );
     }
   }
@@ -1203,7 +1203,9 @@ function define( model ) {
     const type = first?.type || first?.items?.type; // this sequence = no derived type
     const path = type?.path;
     if (path?.length === 1 && path[0]?.id === '$self') { // TODO: no where: ?
-      const { $self } = main.$tableAliases;
+      const $self = main.$tableAliases?.$self ||
+                    main.kind === 'extend' && { name: { id: '$self' } };
+      // remark: an extend has no "table alias" `$self` (relevant for parse-cdl)
       setLink( type, '_artifact', $self );
       setLink( path[0], '_artifact', $self );
     }

package/lib/compiler/extend.js

@@ -27,6 +27,7 @@ const {
 const layers = require('./moduleLayers');
 const { CompilerAssertion } = require('../base/error');
 const { Location } = require('../base/location');
+const { typeParameters } = require('./builtins');
 
 const $location = Symbol.for( 'cds.$location' );
 
@@ -69,6 +70,7 @@ function extend( model ) {
     createRemainingAnnotateStatements,
     extendArtifactBefore,
     extendArtifactAfter,
+    extendForeignKeys,
     applyIncludes,              // TODO: re-check
   } );
 
@@ -91,7 +93,8 @@ function extend( model ) {
   //-----------------------------------------------------------------------------
   // Extensions: general algorithm
   //-----------------------------------------------------------------------------
-  // extendArtifactBefore, extendArtifactAfter, createRemainingAnnotateStatements
+  // extendArtifactBefore, extendArtifactAfter, createRemainingAnnotateStatements,
+  // extendForeignKeys
 
   /**
    * Goes through all (applied) annotations in the given artifact and chooses one
@@ -179,22 +182,53 @@ function extend( model ) {
     else {
       let elementsProp = 'elements';
       if (art.kind !== 'annotate')
-        elementsProp = art.enum && 'enum' || art.foreignKeys && 'foreignKeys' || 'elements';
-      moveDictExtensions( art, extensionsMap, elementsProp, 'elements' );
+        elementsProp = art.enum && 'enum' || art.target && 'foreignKeys' || 'elements';
+
+      // keys are handled in tweak-assocs.js; don't push them down; see extendForeignKeys()
+      if (elementsProp !== 'foreignKeys')
+        moveDictExtensions( art, extensionsMap, elementsProp, 'elements' );
       moveDictExtensions( art, extensionsMap, 'enum' );
     }
   }
 
+  /**
+   * Apply foreign key extensions.  Because foreign keys are handled late in the compiler
+   * (in tweak-assocs.js), we can't apply them in effectiveType(), yet.
+   * Instead, we postpone applying them until all foreign keys were generated.
+   *
+   * @param art
+   */
+  function extendForeignKeys( art ) {
+    // See extendArtifactAfter() for targetAspect/items handling.
+    const sub = art.items || art.targetAspect?.elements && art.targetAspect;
+    if (!art._extensions || sub)
+      return;
+
+    // push down foreign keys
+    moveDictExtensions( art, art._extensions, 'foreignKeys', 'elements' );
+    if (!art.foreignKeys)
+      return;
+
+    forEachGeneric(art, 'foreignKeys', (key) => {
+      if (!key._effectiveType)
+        throw new CompilerAssertion('foreign key should have been processed');
+      extendArtifactBefore( key );
+      extendArtifactAfter( key );
+    });
+  }
+
   /**
    * Applying extensions is handled in extendArtifactAfter(). And only afterward,
    * an effective sequence number is set.  Meaning that if a sub-artifact already
    * has a sequence number, then extensions would be lost.
+   *
+   * A special case are foreign keys, see extendForeignKeys().
    */
   function ensureArtifactNotProcessed( art ) {
     if (!model.options.testMode)
       return;
 
-    if (art.$effectiveSeqNo !== 0 && art.$effectiveSeqNo !== undefined) {
+    if (art.kind !== 'key' && art.$effectiveSeqNo !== 0 && art.$effectiveSeqNo !== undefined) {
       // if the artifact already has a sequence number, then
       // extendArtifactAfter() was already called -> annotations would be lost.
       throw new CompilerAssertion('artifact already processed; extensions would be lost');
@@ -357,7 +391,7 @@ function extend( model ) {
 
   function extensionOverwrites( ext, prop ) {
     return (prop.charAt(0) !== '@')
-      ? [ 'doc', 'length', 'precision', 'scale', 'srid' ].includes( prop )
+      ? (prop === 'doc' || typeParameters.list.includes(prop))
       : !annotationHasEllipsis( ext[prop] );
   }
 
@@ -437,7 +471,7 @@ function extend( model ) {
         query.columns.push( ...ext.columns );
       initSelectItems( query, ext.columns, query, true );
     }
-    else if ([ 'length', 'precision', 'scale', 'srid' ].includes( prop )) {
+    else if (typeParameters.list.includes( prop )) {
       const typeExts = art.$typeExts || (art.$typeExts = {});
       typeExts[prop] = ext;
     }
@@ -647,6 +681,7 @@ function extend( model ) {
     const extensions = extensionsMap[extProp];
     if (!extensions)
       return;
+
     const artDict = art[artProp] || annotateFor( art, extProp ); // no auto-correction in annotate
 
     for (const ext of extensions) {
@@ -762,10 +797,11 @@ function extend( model ) {
     const dict = parent[prop];
     if (!dict) {
       // TODO: check - for each name? - better locations
-      const location = ext._parent[prop][$location] || ext.name.location;
+      const location = ext._parent[prop]?.[$location] || ext.name.location;
       // Remark: no `elements` dict location with `annotate Main:elem`
       switch (prop) {
         // TODO: change texts, somehow similar to checkDefinitions() ?
+        case 'foreignKeys':
         case 'elements':
         case 'enum':            // TODO: extra?
           warning( 'anno-unexpected-elements', [ location, ext._parent ],

package/lib/compiler/index.js

@@ -34,7 +34,7 @@ const { Location, emptyWeakLocation } = require('../base/location');
 const { createMessageFunctions, deduplicateMessages } = require('../base/messages');
 const { checkRemovedDeprecatedFlags } = require('../base/model');
 const { promiseAllDoNotRejectImmediately } = require('../base/node-helpers');
-const { cdsFs } = require('../utils/file');
+const { cdsFs, fileExtension } = require('../utils/file');
 
 const fs = require('fs');
 const path = require('path');
@@ -82,7 +82,7 @@ class ArgumentError extends Error {
 function parseX( source, filename, options = {}, messageFunctions = null ) {
   if (!messageFunctions)
     messageFunctions = createMessageFunctions( options, 'parse' );
-  const ext = path.extname( filename ).slice(1).toLowerCase();
+  const ext = fileExtension( filename );
   const parser = parserForFile( source, ext, options );
   if (parser)
     return parser( source, filename, options, messageFunctions );
@@ -166,10 +166,10 @@ function compileX( filenames, dir = '', options = {}, fileCache = Object.create(
     .then( () => promiseAllDoNotRejectImmediately( input.files.map( readAndParse ) ) )
     .then( testInvocation, (reason) => {
       // do not reject with PromiseAllError, use InvocationError:
-      const errs = reason.valuesOrErrors.filter( e => e instanceof Error );
+      const errs = reason.valuesOrErrors?.filter( e => e instanceof Error ) || [ reason ];
       // internal error if no file IO error (has property `path`)
       return Promise.reject( errs.find( e => !e.path ) ||
-                             new InvocationError( [ ...input.repeated, ...errs ]) );
+                             new InvocationError( [ ...(input?.repeated || []), ...errs ]) );
     } );
 
   if (!options.parseOnly && !options.parseCdl)

package/lib/compiler/populate.js

@@ -83,6 +83,7 @@ function populate( model ) {
     effectiveType,
     getOrigin,
     getInheritedProp,
+    mergeSpecifiedForeignKeys,
   } );
   // let depth = 100;
 
@@ -115,7 +116,10 @@ function populate( model ) {
 
   /** Make sure that effectiveType() is called on all members and items */
   function traverseElementEnvironments( art ) {
-    // TODO: we could leave out foreign keys (but they are traversed via forEachMember)
+    // We leave out foreign keys (as they are traversed via forEachMember).
+    // Keys are handled in tweak-assocs.js
+    if (art.kind === 'key')
+      return;
     let type = effectiveType( art );
     while (type?.items)
       type = effectiveType( type.items );
@@ -529,9 +533,6 @@ function populate( model ) {
       if (!selem) {
         info( 'query-missing-element', [ ielem.name.location, art ], {
           '#': ielem.kind === 'enum' ? 'enum' : 'std', id,
-        }, {
-          std: 'Element $(ID) is missing in specified elements',
-          enum: 'Enum $(ID) is missing in specified enum values',
         } );
       }
       else {
@@ -560,6 +561,8 @@ function populate( model ) {
           setLink( ielem, 'elements$', selem.elements );
         if (selem.enum)
           setLink( ielem, 'enum$', selem.enum );
+        if (selem.foreignKeys)
+          setLink( ielem, 'foreignKeys$', selem.foreignKeys );
       }
     }
 
@@ -574,8 +577,57 @@ function populate( model ) {
       specifiedElement.$isSpecifiedElement = true;
       if (!specifiedElement.$replacement) {
         const loc = [ specifiedElement.name.location, specifiedElement ];
-        error( 'query-unspecified-element', loc, { id },
-               'Element $(ID) does not result from the query' );
+        error( 'query-unspecified-element', loc, { id } );
+      }
+    }
+  }
+
+  /**
+   * Merge _specified_ foreign keys with _inferred_ foreign keys in the given view/element,
+   * where specified elements can appear through CSN.
+   *
+   * We only copy annotations.
+   *
+   * This is important to ensure re-compilability.
+   *
+   * TODO: make this part of a revamped on-demand 'extend' functionality.
+   *
+   * @param art
+   */
+  function mergeSpecifiedForeignKeys( art ) {
+    if (!art.foreignKeys)
+      return; // TODO: Warn if there are no foreign keys?
+
+    let wasAnnotated = false;
+
+    for (const id in art.foreignKeys) {
+      const ielem = art.foreignKeys[id];  // inferred element
+      const selem = art.foreignKeys$[id]; // specified element
+      if (!selem) {
+        info( 'query-missing-element', [ ielem.name.location, art ], { '#': 'foreignKeys', id } );
+      }
+      else {
+        for (const prop in selem) {
+          // just annotation assignments and doc comments for foreign keys
+          if (prop.charAt(0) === '@' || prop === 'doc') {
+            ielem[prop] = selem[prop];
+            // required for gensrc mode of to-csn.js, otherwise the annotation
+            // may be lost during recompilation.
+            ielem[prop].$priority = 'annotate';
+            wasAnnotated = true;
+          }
+        }
+        selem.$replacement = true;
+      }
+    }
+    if (wasAnnotated)
+      setExpandStatusAnnotate( art, 'annotate' );
+
+    for (const id in art.foreignKeys$) {
+      const specifiedElement = art.foreignKeys$[id];
+      if (!specifiedElement.$replacement) {
+        const loc = [ specifiedElement.name.location, specifiedElement ];
+        error( 'query-unspecified-element', loc, { '#': 'foreignKeys', id } );
       }
     }
   }
@@ -816,15 +868,10 @@ function populate( model ) {
     const excludingDict = (colParent || query).excludingDict || Object.create( null );
 
     const envParent = wildcard._pathHead; // TODO: rename _pathHead to _columnParent
-    // console.log('S1:',location.line,location.col,
-    //             envParent&&!!envParent._origin&&envParent._origin.name)
     const env = wildcardColumnEnv( wildcard, query );
     if (!env)
       return;
 
-    // if (envParent) console.log('S2:',location.line,location.col,
-    //             envParent?.name,envParent?._origin?.name,
-    //             Object.keys(env),Object.keys(elements))
     for (const name in env) {
       const navElem = env[name];
       // TODO: remove all access to masked (use 'grep')

package/lib/compiler/propagator.js

@@ -49,12 +49,17 @@ function propagate( model ) {
     targetAspect,
     cardinality: notWithExpand,
     on: notWithExpand,
-    foreignKeys: expensive,    // includes "notWithExpand", dictionary copy
+    // "expensive" includes "notWithExpand"
+    // required for places where we don't handle associations, such as in parameters;
+    // otherwise already expanded and rewritten.
+    foreignKeys: expensive,
     items,
+    // required for propagation in targetAspect; otherwise already expanded
     elements: expensive,
-    enum: expensive,
-    params: expensive,          // actually only with parent action
-    returns,
+    // already expanded if necessary
+    //   enum: expensive,
+    //   params: expensive,          // actually only with parent action
+    //   returns,
     $enclosed: annotation,
   };
   const ruleToFunction = {