@malloydata/malloy 0.0.56-dev230710182711 → 0.0.56-dev230710205111

package/dist/lang/test/parse.spec.js

@@ -52,9 +52,10 @@ describe('model statements', () => {
             });
             expect(m).translationToFailWith('a1 is not a connection');
         });
-        // TODO unskip this when ENABLE_M4_WARNINGS becomes a document annotation
-        test.skip('table function is deprecated', () => {
-            expect("testA is table('conn:aTable')").toTranslateWithWarnings("`table('connection_name:table_path')` is deprecated; use `connection_name.table('table_path')`");
+        test('table function is deprecated', () => {
+            expect(`##! m4warnings
+      source: testA is table('conn:aTable')
+    `).toTranslateWithWarnings("`table('connection_name:table_path')` is deprecated; use `connection_name.table('table_path')`");
         });
     });
     describe('source:', () => {
@@ -88,13 +89,12 @@ describe('model statements', () => {
         });
     });
     describe('query:', () => {
-        // Delete this when ENABLE_M4_WARNINGS is converted to an annotation
         test('anonymous query', () => {
             expect((0, test_translator_1.markSource) `query: ${"table('aTable') -> { group_by: astr }"}`).toTranslate();
         });
-        // Unskip this when ENABLE_M4_WARNINGS is converted to an annotation
-        test.skip('anonymous query', () => {
-            expect((0, test_translator_1.markSource) `query: ${"table('aTable') -> { group_by: astr }"}`).toTranslateWithWarnings('Anonymous `query:` statements are deprecated, use `run:` instead');
+        test('anonymous query', () => {
+            expect((0, test_translator_1.markSource) `##! m4warnings
+        query: ${"conn.table('aTable') -> { group_by: astr }"}`).toTranslateWithWarnings('Anonymous `query:` statements are deprecated, use `run:` instead');
         });
         test('run query', () => expect("run: table('aTable') -> { group_by: astr }").toTranslate());
         test('run query ref', () => expect(`
@@ -1018,16 +1018,18 @@ describe('qops', () => {
             expect('query: a -> { extend: { dimension: x is 1; dimension: y is 2; measure: c is count() }; group_by: x, y; aggregate: c }').toTranslate();
         });
     });
-    // TODO ENABLE_M4_WARNINGS: unskip when we have an M4 warning flag
-    describe.skip('declare/query join warnings', () => {
+    describe('declare/query join warnings', () => {
         test('declare warning in query', () => {
-            expect((0, test_translator_1.markSource) `query: a -> { declare: ${'x is 1'}; group_by: x, y }`).toTranslateWithWarnings('`declare:` is deprecated; use `dimension:` or `measure:` inside a source or `extend:` block');
+            expect((0, test_translator_1.markSource) `##! m4warnings
+        run: a -> { ${'declare: x is 1'}; group_by: x }`).toTranslateWithWarnings('`declare:` is deprecated; use `dimension:` or `measure:` inside a source or `extend:` block');
         });
         test('declare warning in source', () => {
-            expect((0, test_translator_1.markSource) `source: a2 is a { declare: ${'x is 1'} }`).toTranslateWithWarnings('`declare:` is deprecated; use `dimension:` or `measure:` inside a source or `extend:` block');
+            expect((0, test_translator_1.markSource) `##! m4warnings
+        source: a2 is a extend { ${'declare: x is 1'} }`).toTranslateWithWarnings('`declare:` is deprecated; use `dimension:` or `measure:` inside a source or `extend:` block');
         });
         test('joins in query', () => {
-            expect((0, test_translator_1.markSource) `query: a -> { ${'join_one: b on true'}; ${'join_many: c is b on true'}; ${'join_cross: d is b on true'}; group_by: b.astr }`).toTranslateWithWarnings('Joins in queries are deprecated, move into an `extend:` block.', 'Joins in queries are deprecated, move into an `extend:` block.', 'Joins in queries are deprecated, move into an `extend:` block.');
+            expect((0, test_translator_1.markSource) `##! m4warnings
+        run: a -> { ${'join_one: b on true'}; ${'join_many: c is b on true'}; ${'join_cross: d is b on true'}; group_by: b.astr }`).toTranslateWithWarnings('Joins in queries are deprecated, move into an `extend:` block.', 'Joins in queries are deprecated, move into an `extend:` block.', 'Joins in queries are deprecated, move into an `extend:` block.');
         });
     });
     test('refine query with extended source', () => {
@@ -1593,18 +1595,18 @@ describe('error handling', () => {
     `).translationToFailWith("Cannot redefine 'airports'");
     });
     test('query from undefined source', () => {
-        expect((0, test_translator_1.markSource) `query: ${'x'}->{ project: y }`).translationToFailWith("Undefined source 'x'");
+        expect((0, test_translator_1.markSource) `query: ${'x'}->{ project: y }`).translationToFailWith("Undefined query or source 'x'");
     });
     test('query with expression from undefined source', () => {
         // Regression check: Once upon a time this died with an exception even
         // when "query: x->{ group_by: y}" (above) generated the correct error.
-        expect((0, test_translator_1.markSource) `query: ${'x'}->{ project: y is z / 2 }`).translationToFailWith("Undefined source 'x'");
+        expect((0, test_translator_1.markSource) `query: ${'x'}->{ project: y is z / 2 }`).translationToFailWith("Undefined query or source 'x'");
     });
     test('join reference before definition', () => {
         expect((0, test_translator_1.markSource) `
         source: newAB is a { join_one: newB is ${'bb'} on astring }
         source: newB is b
-      `).translationToFailWith("Undefined source 'bb'");
+      `).translationToFailWith("Undefined query or source 'bb'");
     });
     test('non-rename rename', () => {
         expect('source: na is a { rename: astr is astr }').translationToFailWith("Can't rename field to itself");
@@ -2895,9 +2897,9 @@ describe('sql expressions', () => {
             expect(m).toTranslate();
         }
     });
-    // TODO unskip when ENABLE_M4_WARNINGS is turned into an annotation
-    test.skip('sql statement deprecation warning', () => {
-        const m = new test_translator_1.TestTranslator('sql: bad_sql is {select: """SELECT 1 as one"""}');
+    test('sql statement deprecation warning', () => {
+        const m = new test_translator_1.TestTranslator(`##! m4warnings
+      sql: bad_sql is {select: """SELECT 1 as one"""}`);
         const req = m.translate().compileSQL;
         if (req) {
             m.update({
@@ -2906,5 +2908,252 @@ describe('sql expressions', () => {
         }
         expect(m).toTranslateWithWarnings('`sql:` statement is deprecated, use `connection_name.sql(...)` instead');
     });
+    test('reference to sql expression in run', () => {
+        const m = new test_translator_1.TestTranslator(`
+      run: bigquery.sql("""select 1 as one""")
+    `);
+        expect(m).toParse();
+        const compileSql = m.translate().compileSQL;
+        expect(compileSql).toBeDefined();
+        if (compileSql) {
+            m.update({
+                compileSQL: { [compileSql.name]: getSelectOneStruct(compileSql) },
+            });
+            expect(m).toTranslate();
+        }
+    });
+    test('reference to sql expression in query def', () => {
+        const m = new test_translator_1.TestTranslator(`
+      query: q is bigquery.sql("""select 1 as one""")
+    `);
+        expect(m).toParse();
+        const compileSql = m.translate().compileSQL;
+        expect(compileSql).toBeDefined();
+        if (compileSql) {
+            m.update({
+                compileSQL: { [compileSql.name]: getSelectOneStruct(compileSql) },
+            });
+            expect(m).toTranslate();
+        }
+    });
+    test('reference to sql expression in anonymous query', () => {
+        const m = new test_translator_1.TestTranslator(`
+      query: bigquery.sql("""select 1 as one""")
+    `);
+        expect(m).toParse();
+        const compileSql = m.translate().compileSQL;
+        expect(compileSql).toBeDefined();
+        if (compileSql) {
+            m.update({
+                compileSQL: { [compileSql.name]: getSelectOneStruct(compileSql) },
+            });
+            expect(m).toTranslate();
+        }
+    });
+    // TODO this is not possible to implement yet unless we
+    // can distinguish between the generated IR of `conn.sql(...)`
+    // and `conn.sql(...) -> { project: * }`.
+    test.skip('cannot refine a SQL query', () => {
+        const m = new test_translator_1.TestTranslator(`
+      query: q1 is bigquery.sql("""select 1 as one""")
+      run: q1 refine { where: 1 = 1 }
+    `);
+        expect(m).toParse();
+        const compileSql = m.translate().compileSQL;
+        expect(compileSql).toBeDefined();
+        if (compileSql) {
+            m.update({
+                compileSQL: { [compileSql.name]: getSelectOneStruct(compileSql) },
+            });
+            expect(m).translationToFailWith('Cannot refine a SQL query');
+        }
+    });
+});
+describe('extend and refine', () => {
+    describe('extend and refine, new syntax', () => {
+        test('query name with query refinements', () => {
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q refine { group_by: ai2 is ai }
+        `).toTranslate();
+        });
+        test('query name with source refinements', () => {
+            expect(`
+        query: q is a -> { group_by: ai }
+        source: s is q extend { dimension: ai_2 is ai + ai }
+        `).toTranslate();
+        });
+        test('source name with query refinements', () => {
+            expect('run: a refine { group_by: ai }').translationToFailWith("Illegal reference to 'a', query expected");
+        });
+        test('source name with source refinements', () => {
+            expect('source: s is a extend { dimension: ai_2 is ai + ai }').toTranslate();
+        });
+        test('query with source refinements', () => {
+            expect('source: s is a -> { group_by: ai } extend { dimension: ai_2 is ai + ai }').toTranslate();
+        });
+        test('query with extension then new stage', () => {
+            expect('run: a { dimension: x is 1 } -> { group_by: x }').toTranslate();
+        });
+    });
+    describe('extend and refine, old syntax', () => {
+        test('query name with query refinements', () => {
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { group_by: ai2 is ai }
+        `).toTranslate();
+        });
+        test('query name with source refinements', () => {
+            expect(`
+        query: q is a -> { group_by: ai }
+        source: s is q { dimension: ai_2 is ai + ai }
+        `).toTranslate();
+        });
+        test('source name with query refinements', () => {
+            expect('run: a { group_by: one }').translationToFailWith("Illegal reference to 'a', query expected");
+        });
+        test('source name with source refinements', () => {
+            expect('source: s is a { dimension: ai_2 is ai + ai }').toTranslate();
+        });
+        test('source name with ambiguous refinements', () => {
+            // Ambiguous refinements are assumed to be source extensions
+            expect('run: a { join_one: b on b.ai = ai } -> { project: b.* }').toTranslate();
+            expect('run: a { where: 1 = 1 } -> { project: * }').toTranslate();
+            expect('run: a { declare: three is 3 } -> { project: * }').toTranslate();
+            expect('source: s is a { join_one: b on b.ai = ai }').toTranslate();
+            expect('source: s is a { where: 1 = 1 }').toTranslate();
+            expect('source: s is a { declare: three is 3 }').toTranslate();
+        });
+        test('query name with ambiguous refinements', () => {
+            // Here we implicitly convert the query into a source.
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { join_one: b on 1 = 1 } -> { project: b.* }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { where: 1 = 1 } -> { project: * }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { declare: three is 3 } -> { project: * }
+      `).toTranslate();
+            // Without the new stage we automatically recognize this as a query refinement
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { join_one: b on 1 = 1 }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { where: 1 = 1 }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { declare: three is 3 }
+      `).toTranslate();
+            // Can also just add from() to use as a source
+            // TODO add a warning when you use from() -- "`from()` is deprecated; to apply source extensions to a query, use `extend`"
+            expect(`
+        query: q is a -> { group_by: ai }
+        source: s is from(q) { join_one: b on b.ai = ai }
+      `).toTranslate();
+            // Can also add an arrow to clarify that it's a query
+            // Of course, number 1 and 3 are actually useless because you're defining
+            // something and then not using it...
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: -> q { join_one: b on b.ai = ai } -> { project: * }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: -> q { where: 1 = 1 } -> { project: * }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: -> q { declare: three is 3 } -> { project: * }
+      `).toTranslate();
+            // Alternatively, fix 1 and 3 by actually using the declared thing
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { join_one: b on b.ai = ai; group_by: ai2 is b.ai }
+      `).toTranslate();
+            expect(`
+        query: q is a -> { group_by: ai }
+        run: q { declare: three is 3; group_by: three }
+      `).toTranslate();
+        });
+    });
+    describe('extend and refine fallout', () => {
+        test('syntactically valid to run a source, but still illegal', () => {
+            expect('run: a').translationToFailWith("Illegal reference to 'a', query expected");
+        });
+        test('syntactically valid to refine a source, but illegal', () => {
+            expect('run: a { group_by: ai } ').translationToFailWith("Illegal reference to 'a', query expected");
+            expect('run: a refine { group_by: ai } ').translationToFailWith("Illegal reference to 'a', query expected");
+        });
+    });
+    describe('turtles', () => {
+        test('explicit refine in turtle works', () => {
+            expect(`source: c is a extend {
+        query: x is { project: * }
+        query: y is x refine { limit: 1 }
+      }`).toTranslate();
+        });
+        test('implicit refine in turtle works', () => {
+            expect(`source: c is a extend {
+        query: x is { project: * }
+        query: y is x { limit: 1 }
+      }`).toTranslate();
+        });
+    });
+    describe('nests', () => {
+        test('explicit refine in nest', () => {
+            expect(`source: c is a extend {
+        query: x is { project: * }
+      }
+
+      run: c -> {
+        nest: x refine { limit: 1 }
+      }`).toTranslate();
+        });
+        test('implicit refine in nest', () => {
+            expect(`source: c is a extend {
+        query: x is { project: * }
+      }
+
+      run: c -> {
+        nest: x { limit: 1 }
+      }`).toTranslate();
+        });
+    });
+    describe('m4 warnings', () => {
+        test('implicit refine in nest', () => {
+            expect(`##! m4warnings
+      source: c is a extend {
+        query: x is { project: * }
+      }
+
+      run: c -> {
+        nest: x { limit: 1 }
+      }`).toTranslateWithWarnings('Implicit query refinement is deprecated, use the `refine` operator.');
+        });
+        test('implicit refine in turtle works', () => {
+            expect(`##! m4warnings
+      source: c is a extend {
+        query: x is { project: * }
+        query: y is x { limit: 1 }
+      }`).toTranslateWithWarnings('Implicit query refinement is deprecated, use the `refine` operator.');
+        });
+        test('implicit query refinement', () => {
+            expect(`##! m4warnings
+        query: q is a -> { group_by: ai }
+        run: q { group_by: three is 3 }
+      `).toTranslateWithWarnings('Implicit query refinement is deprecated, use the `refine` operator.');
+        });
+        test('implicit source extension', () => {
+            expect(`##! m4warnings
+        source: s is a { dimension: ai_2 is ai + ai }`).toTranslateWithWarnings('Implicit source extension is deprecated, use the `extend` operator.');
+        });
+    });
 });
 //# sourceMappingURL=parse.spec.js.map

package/dist/tags.d.ts

@@ -18,4 +18,5 @@ export declare class Tags {
     getMalloyTags(): MalloyTags;
     getTagList(): string[];
 }
+export declare function parseTagProperties(src: string, tagProp: MalloyTagProperties): MalloyTagProperties | undefined;
 export {};

package/dist/tags.js

@@ -22,7 +22,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Tags = void 0;
+exports.parseTagProperties = exports.Tags = void 0;
 function isDocTag(a) {
     return (a === null || a === void 0 ? void 0 : a.docString) !== undefined;
 }
@@ -79,6 +79,12 @@ function parseTag(src, tagProp) {
     if (!propMatch) {
         return;
     }
+    const newProps = parseTagProperties(src.slice(propMatch[0].length), tagProp);
+    if (newProps) {
+        return { properties: newProps };
+    }
+}
+function parseTagProperties(src, tagProp) {
     /*
      * I went back and forth if the grammar for these annotations should be
      * in the parser or the lexer. Eventually the fact that the lexer ate
@@ -86,7 +92,7 @@ function parseTag(src, tagProp) {
      *
      * Seems wrong to be writing a parser though.
      */
-    const tokens = tokenize(src.slice(propMatch[0].length));
+    const tokens = tokenize(src);
     let tn = 0;
     const lastToken = tokens.length - 1;
     while (tn <= lastToken) {
@@ -119,8 +125,9 @@ function parseTag(src, tagProp) {
         }
         tn += 1;
     }
-    return { properties: tagProp };
+    return tagProp;
 }
+exports.parseTagProperties = parseTagProperties;
 function tokenize(src) {
     const parts = [];
     src = src.trim();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.56-dev230710182711",
+  "version": "0.0.56-dev230710205111",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",