@sasjs/utils 2.49.0 → 2.49.2

package/macros/mp_getddl.sas

@@ -0,0 +1,400 @@
+/**
+  @file mp_getddl.sas
+  @brief Extract DDL in various formats, by table or library
+  @details Data Definition Language relates to a set of SQL instructions used
+    to create tables in SAS or a database.  The macro can be used at table or
+    library level.  The default behaviour is to create DDL in SAS format.
+
+    Note - views are not currently supported.
+
+  Usage:
+
+      data test(index=(pk=(x y)/unique /nomiss));
+        x=1;
+        y='blah';
+        label x='blah';
+      run;
+      proc sql; describe table &syslast;
+      %mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mp_getconstraints.sas
+
+  @param lib libref of the library to create DDL for.  Should be assigned.
+  @param ds dataset to create ddl for (optional)
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
+  @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
+  @param showlog= Set to YES to show the DDL in the log
+  @param schema= Choose a preferred schema name (default is to use actual schema
+    ,else libref)
+  @param applydttm= for non SAS DDL, choose if columns are created with native
+    datetime2 format or regular decimal type
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
+%end;
+
+%if %length(&libref)=0 %then %let libref=WORK;
+%let flavour=%upcase(&flavour);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  ;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname)="%upcase(&libref)"
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  ;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(memname) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname)="%upcase(&libref)"
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  order by idxusage, indxname, indxpos
+  ;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%upcase(&libref),ds=%upcase(&ds),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%macro addConst();
+  %global constraints_used;
+  data _null_;
+    length ctype $11 constraint_name_orig $256 constraints_used $5000;
+    set &colconst(
+        where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))
+      ) end=last;
+    file &fref mod;
+    by constraint_type constraint_name;
+    retain constraints_used;
+    constraint_name_orig=constraint_name;
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='PRIMARY KEY';
+    else ctype=strip(constraint_type);
+    %if &flavour=TSQL %then %do;
+      column_name=catt('[',column_name,']');
+      constraint_name=catt('[',constraint_name,']');
+    %end;
+    %else %if &flavour=PGSQL %then %do;
+      column_name=catt('"',column_name,'"');
+      constraint_name=catt('"',constraint_name,'"');
+    %end;
+    if first.constraint_name then do;
+      constraints_used = catx(' ', constraints_used, constraint_name_orig);
+      put "   ,CONSTRAINT " constraint_name ctype "(" ;
+      put '     ' column_name;
+    end;
+  else put '     ,' column_name;
+  if last.constraint_name then do;
+    put "   )";
+    call symput('constraints_used',strip(constraints_used));
+  end;
+  run;
+  %put &=constraints_used;
+%mend addConst;
+
+data _null_;
+  file &fref mod;
+  put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
+run;
+
+%local x curds;
+%if &flavour=SAS %then %do;
+  %do x=1 %to %sysfunc(countw(&dsnlist));
+    %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
+    data _null_;
+      file &fref mod;
+      length lab $1024 typ $20;
+      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+
+      if _n_=1 then do;
+        if memtype='DATA' then do;
+          put "create table &libref..&curds(";
+        end;
+        else do;
+          /* just a placeholder - we filter out views at the top */
+          put "create view &libref..&curds(";
+        end;
+        put "    "@@;
+      end;
+      else put "   ,"@@;
+      if length(format)>1 then fmt=" format="!!cats(format);
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
+      if notnull='yes' then notnul=' not null';
+      if type='char' then typ=cats('char(',length,')');
+      else if length ne 8 then typ='num length='!!cats(length);
+      else typ='num';
+      put name typ fmt notnul lab;
+    run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
+    data _null_;
+      file &fref mod;
+      put ');';
+    run;
+
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+              )
+          )
+        );
+      file &fref mod;
+      by idxusage indxname;
+/*       ds=cats(libname,'.',memname); */
+      if first.indxname then do;
+          put 'CREATE UNIQUE INDEX ' indxname "ON &libref..&curds (" ;
+          put '  ' name ;
+      end;
+      else put '  ,' name ;
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
+      end;
+    run;
+
+/*
+    ods output IntegrityConstraints=ic;
+    proc contents data=testali out2=info;
+    run;
+    */
+  %end;
+%end;
+%else %if &flavour=TSQL %then %do;
+  /* if schema does not exist, set to be same as libref */
+  %local schemaactual;
+  proc sql noprint;
+  select sysvalue into: schemaactual
+    from dictionary.libnames
+    where upcase(libname)="&libref" and engine='SQLSVR';
+  %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
+
+  %do x=1 %to %sysfunc(countw(&dsnlist));
+    %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* TSQL Flavour DDL for &schema..&curds */";
+    data _null_;
+      file &fref mod;
+      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+      if _n_=1 then do;
+        if memtype='DATA' then do;
+          put "create table [&schema].[&curds](";
+        end;
+        else do;
+          /* just a placeholder - we filter out views at the top */
+          put "create view [&schema].[&curds](";
+        end;
+        put "    "@@;
+      end;
+      else put "   ,"@@;
+      format=upcase(format);
+      if 1=0 then; /* dummy if */
+      %if &applydttm=YES %then %do;
+        else if format=:'DATETIME' then fmt='[datetime2](7)  ';
+      %end;
+      else if type='num' then fmt='[decimal](18,2)';
+      else if length le 8000 then fmt='[varchar]('!!cats(length)!!')';
+      else fmt=cats('[varchar](max)');
+      if notnull='yes' then notnul=' NOT NULL';
+      put "[" name +(-1) "]" fmt notnul;
+    run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+          )
+        )
+      );
+      file &fref mod;
+      by idxusage indxname;
+      *ds=cats(libname,'.',memname);
+      if first.indxname then do;
+        /* add nonclustered in case of multiple unique indexes */
+        put '   ,index [' indxname +(-1) '] UNIQUE NONCLUSTERED (';
+        put '     [' name +(-1) ']';
+      end;
+      else put '     ,[' name +(-1) ']';
+      if last.indxname then do;
+        put '   )';
+      end;
+    run;
+
+    data _null_;
+      file &fref mod;
+      put ')';
+      put 'GO';
+    run;
+
+    /* add extended properties for labels */
+    data _null_;
+      file &fref mod;
+      length nm $64 lab $1024;
+      set &colinfo (where=(upcase(memname)="&curds" and label ne '')) end=last;
+      nm=cats("N'",tranwrd(name,"'","''"),"'");
+      lab=cats("N'",tranwrd(label,"'","''"),"'");
+      put ' ';
+      put "EXEC sys.sp_addextendedproperty ";
+      put "  @name=N'MS_Description',@value=" lab ;
+      put "  ,@level0type=N'SCHEMA',@level0name=N'&schema' ";
+      put "  ,@level1type=N'TABLE',@level1name=N'&curds'";
+      put "  ,@level2type=N'COLUMN',@level2name=" nm ;
+      if last then put 'GO';
+    run;
+  %end;
+%end;
+%else %if &flavour=PGSQL %then %do;
+  /* if schema does not exist, set to be same as libref */
+  %local schemaactual;
+  proc sql noprint;
+  select sysvalue into: schemaactual
+    from dictionary.libnames
+    where upcase(libname)="&libref" and engine='POSTGRES';
+  %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
+  data _null_;
+    file &fref mod;
+    put "CREATE SCHEMA &schema;";
+  %do x=1 %to %sysfunc(countw(&dsnlist));
+    %let curds=%scan(&dsnlist,&x);
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
+        end;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
+
+      /* Extra step for data constraints */
+      %addConst()
+
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
+
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
+          )
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
+          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
+          put '  "' name +(-1) '"' ;
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
+  %end;
+%end;
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_getddl;

package/macros/mp_getformats.sas

@@ -0,0 +1,124 @@
+/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_applyformats.sas
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  %mddl_sas_cntlout(libds=&outdetail)
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    data &tempds;
+      if 0 then set &outdetail;
+      set &tempds;
+    run;
+    proc append base=&outdetail data=&tempds ;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;

package/macros/mp_getmaxvarlengths.sas

@@ -0,0 +1,130 @@
+/**
+  @file
+  @brief Scans a dataset to find the max length of the variable values
+  @details
+  This macro will scan a base dataset and produce an output dataset with two
+  columns:
+
+  - NAME    Name of the base dataset column
+  - MAXLEN  Maximum length of the data contained therein.
+
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
+
+  Usage:
+
+      %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
+
+  <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+  @li mf_getvarformat.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
+**/
+
+%macro mp_getmaxvarlengths(
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
+)/*/STORE SOURCE*/;
+
+%local vars prefix x var fmt srcds;
+%let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
+
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
+
+proc sql;
+create table &outds (rename=(
+    %do x=1 %to %sysfunc(countw(&vars,%str( )));
+      &prefix.&x=%scan(&vars,&x)
+    %end;
+    ))
+  as select
+    %do x=1 %to %sysfunc(countw(&vars,%str( )));
+      %let var=%scan(&vars,&x);
+      %if &x>1 %then ,;
+      %if %mf_getvartype(&libds,&var)=C %then %do;
+        max(lengthn(&var)) as &prefix.&x
+      %end;
+      %else %if &num2char=YES %then %do;
+        %let fmt=%mf_getvarformat(&libds,&var);
+        %put fmt=&fmt;
+        %if %str(&fmt)=%str() %then %do;
+          max(lengthn(cats(&var))) as &prefix.&x
+        %end;
+        %else %do;
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
+        %end;
+      %end;
+      %else %do;
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
+        %end;
+      %end;
+    %end;
+  from &srcds;
+
+  proc transpose data=&outds
+    out=&outds(rename=(_name_=NAME COL1=MAXLEN));
+  run;
+
+%mend mp_getmaxvarlengths;